You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@wayang.apache.org by gi...@apache.org on 2024/09/22 10:40:15 UTC
(incubator-wayang-website) branch deploy updated: deploy: d5c5667ec8d66448843e73f437bfe890eeac02c9
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch deploy
in repository https://gitbox.apache.org/repos/asf/incubator-wayang-website.git
The following commit(s) were added to refs/heads/deploy by this push:
new 77b5f6dc deploy: d5c5667ec8d66448843e73f437bfe890eeac02c9
77b5f6dc is described below
commit 77b5f6dc653598267a1450285616305bd48f9b26
Author: kamir <ka...@users.noreply.github.com>
AuthorDate: Sun Sep 22 10:40:09 2024 +0000
deploy: d5c5667ec8d66448843e73f437bfe890eeac02c9
---
404.html | 4 +-
assets/js/19340771.3a597734.js | 1 -
assets/js/1a7d9bc6.f019a3d1.js | 1 -
assets/js/1f6e4ff1.ec0453c9.js | 1 +
assets/js/237ddcab.4f9f8a9a.js | 1 -
assets/js/29e20dbd.94a6213e.js | 1 +
assets/js/29e20dbd.f0dc9b10.js | 1 -
assets/js/34f284a7.4a7bccac.js | 1 +
assets/js/34f284a7.f6acbe7e.js | 1 -
.../{35a4b3d1.a1b6ca44.js => 35a4b3d1.51b6a3ed.js} | 2 +-
assets/js/3df8af31.8b4f09b3.js | 1 +
assets/js/3df8af31.8c2e6b09.js | 1 -
assets/js/3e5cd75b.8826fd6a.js | 1 +
assets/js/3e5cd75b.8b6d58e0.js | 1 -
.../{47457315.3fd21bbe.js => 47457315.a462d621.js} | 2 +-
assets/js/4d06a287.3d412fc3.js | 1 +
assets/js/4d06a287.e04133cf.js | 1 -
.../{5f842dbc.2b9209c3.js => 5f842dbc.54716e90.js} | 2 +-
.../{720d5f7a.af3b91c3.js => 720d5f7a.d6da4f1e.js} | 2 +-
assets/js/75f5c5f6.10d21cb5.js | 1 -
assets/js/75f5c5f6.b5df1a15.js | 1 +
.../{77806f28.e7c413b3.js => 77806f28.6901a0cf.js} | 2 +-
.../{7c80c937.bff51920.js => 7c80c937.93fb8865.js} | 2 +-
assets/js/814f3328.5ac9b2c3.js | 1 -
assets/js/814f3328.f9c5cafe.js | 1 +
.../{86331d88.34234831.js => 86331d88.c1ef525c.js} | 2 +-
assets/js/8eb4e46b.d22aff7e.js | 1 -
assets/js/8f973d51.053525a0.js | 1 +
assets/js/8f973d51.da7f5ca3.js | 1 -
assets/js/922da4c8.2fa414fe.js | 1 -
assets/js/935c02bb.25b25a20.js | 1 +
assets/js/935c02bb.f9ab0c3c.js | 1 -
assets/js/a7023ddc.44152c59.js | 1 +
assets/js/a7023ddc.f00d0b14.js | 1 -
assets/js/ae568e66.7a788392.js | 1 +
assets/js/ae568e66.92dd05e4.js | 1 -
assets/js/ae720ac1.174e3afa.js | 1 -
assets/js/ae720ac1.e4462485.js | 1 +
assets/js/b2b675dd.300bb8b4.js | 1 +
assets/js/b2b675dd.b5eff2b2.js | 1 -
assets/js/b2f554cd.389ebf9a.js | 1 +
assets/js/b2f554cd.50606bdc.js | 1 -
.../{bae78078.70745d80.js => bae78078.a537a88f.js} | 2 +-
assets/js/cfab9f0f.3319519a.js | 1 +
assets/js/d2f02a99.3009b741.js | 1 -
assets/js/de9e258c.25d98ce0.js | 1 -
.../{e82464af.d4e009f4.js => e82464af.efc150d0.js} | 2 +-
assets/js/effa2a56.33b72a28.js | 1 +
assets/js/effa2a56.6b6490a2.js | 1 -
assets/js/f5d9e7eb.0f4692b4.js | 1 -
assets/js/f5d9e7eb.1e9fed5e.js | 1 +
assets/js/f9d658b5.98d642bc.js | 1 +
assets/js/f9d658b5.f5b24665.js | 1 -
assets/js/main.0cc2e93a.js | 2 -
assets/js/main.4f5176d9.js | 2 +
...js.LICENSE.txt => main.4f5176d9.js.LICENSE.txt} | 0
assets/js/runtime~main.01bb8940.js | 1 -
assets/js/runtime~main.8489d39d.js | 1 +
blog/archive/index.html | 6 +-
blog/atom.xml | 406 +++++++--------------
blog/index.html | 375 +++++++++----------
blog/kafka-meets-wayang-1/index.html | 18 +-
blog/kafka-meets-wayang-2/index.html | 20 +-
blog/kafka-meets-wayang-3/index.html | 8 +-
blog/page/2/index.html | 63 ----
blog/rss.xml | 394 +++++++-------------
blog/tags/ai/index.html | 6 +-
blog/tags/asf/index.html | 169 ++++++---
.../index.html | 169 +--------
blog/tags/federated/index.html | 6 +-
blog/tags/index.html | 6 +-
blog/tags/kafka/index.html | 169 +--------
blog/tags/ml/index.html | 6 +-
blog/tags/presto/index.html | 6 +-
blog/tags/python/index.html | 6 +-
blog/tags/release/index.html | 169 ++++++---
blog/tags/spark/index.html | 65 +---
blog/tags/tensorflow/index.html | 6 +-
blog/tags/trino/index.html | 6 +-
blog/tags/wayang/index.html | 377 +++++++++----------
blog/tags/wayang/page/2/index.html | 63 ----
blog/wayang-federated-ai/index.html | 6 +-
blog/wayang-python-api/index.html | 8 +-
blog/wayang-release-odysse/index.html | 140 +++++++
blog/wayang-tensorflow/index.html | 8 +-
blog/wayang-vs-trino/index.html | 8 +-
blog/website_update/index.html | 8 +-
docs/community/committer/index.html | 4 +-
docs/community/contribute/index.html | 4 +-
docs/community/mailinglist/index.html | 4 +-
docs/community/repositories/index.html | 4 +-
docs/community/team/index.html | 4 +-
docs/guide/adding-operators/index.html | 4 +-
docs/guide/api-documentation/index.html | 4 +-
docs/guide/developing-in-wayang/index.html | 4 +-
docs/guide/examples/index.html | 4 +-
docs/guide/getting-started/index.html | 4 +-
docs/guide/installation/index.html | 4 +-
docs/guide/ml4all/index.html | 4 +-
docs/guide/optimizer/index.html | 4 +-
docs/guide/wayang-docker/index.html | 4 +-
docs/introduction/about/index.html | 4 +-
docs/introduction/benchmark/index.html | 4 +-
docs/introduction/features/index.html | 4 +-
docs/start/download/index.html | 4 +-
index.html | 4 +-
lunr-index-1726903642488.json | 1 -
lunr-index-1727001579936.json | 1 +
lunr-index.json | 2 +-
search-doc-1726903642488.json | 1 -
search-doc-1727001579936.json | 1 +
search-doc.json | 2 +-
sitemap.xml | 2 +-
113 files changed, 1166 insertions(+), 1688 deletions(-)
diff --git a/404.html b/404.html
index 69f06089..a86ccd42 100644
--- a/404.html
+++ b/404.html
@@ -5,8 +5,8 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">Page Not Found | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/404.html"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docusaurus_tag" content="default"><meta d [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.01bb8940.js" defer="defer"></script>
-<script src="/assets/js/main.0cc2e93a.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.8489d39d.js" defer="defer"></script>
+<script src="/assets/js/main.4f5176d9.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
diff --git a/assets/js/19340771.3a597734.js b/assets/js/19340771.3a597734.js
deleted file mode 100644
index da29f029..00000000
--- a/assets/js/19340771.3a597734.js
+++ /dev/null
@@ -1 +0,0 @@
-"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[1438],{6092:(e,a,n)=>{n.r(a),n.d(a,{assets:()=>i,contentTitle:()=>l,default:()=>h,frontMatter:()=>o,metadata:()=>c,toc:()=>s});var t=n(5893),r=n(1151);const o={slug:"kafka-meets-wayang-1",title:"Apache Kafka meets Apache Wayang - Part 1",authors:"kamir",tags:["wayang","ASF","release"]},l="How I failed to release Apache Wayang",c={permalink:"/blog/kafka-meets-wayang-1",source:"@site/blog/release-odys [...]
\ No newline at end of file
diff --git a/assets/js/1a7d9bc6.f019a3d1.js b/assets/js/1a7d9bc6.f019a3d1.js
deleted file mode 100644
index 96ac632d..00000000
--- a/assets/js/1a7d9bc6.f019a3d1.js
+++ /dev/null
@@ -1 +0,0 @@
-"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[1766],{5625:a=>{a.exports=JSON.parse('{"label":"wayang","permalink":"/blog/tags/wayang","allTagsPath":"/blog/tags","count":12,"unlisted":false}')}}]);
\ No newline at end of file
diff --git a/assets/js/1f6e4ff1.ec0453c9.js b/assets/js/1f6e4ff1.ec0453c9.js
new file mode 100644
index 00000000..0971b038
--- /dev/null
+++ b/assets/js/1f6e4ff1.ec0453c9.js
@@ -0,0 +1 @@
+"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[9262],{213:(e,a,n)=>{n.r(a),n.d(a,{assets:()=>i,contentTitle:()=>l,default:()=>h,frontMatter:()=>o,metadata:()=>c,toc:()=>s});var t=n(5893),r=n(1151);const o={slug:"wayang-release-odysse",title:"Apache Wayang Release Odysse",authors:"kamir",tags:["wayang","ASF","release"]},l="How I failed to release Apache Wayang",c={permalink:"/blog/wayang-release-odysse",source:"@site/blog/2024-09-01-release-odyss [...]
\ No newline at end of file
diff --git a/assets/js/237ddcab.4f9f8a9a.js b/assets/js/237ddcab.4f9f8a9a.js
deleted file mode 100644
index e538cb0a..00000000
--- a/assets/js/237ddcab.4f9f8a9a.js
+++ /dev/null
@@ -1 +0,0 @@
-"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[4251],{1663:(e,a,n)=>{n.r(a),n.d(a,{assets:()=>l,contentTitle:()=>r,default:()=>p,frontMatter:()=>o,metadata:()=>s,toc:()=>c});var t=n(5893),i=n(1151);const o={slug:"kafka-meets-wayang-2",title:"Apache Kafka meets Apache Wayang - Part 3",authors:"kamir",tags:["wayang","kafka","spark","cross organization data collaboration"]},r=void 0,s={permalink:"/blog/kafka-meets-wayang-2",source:"@site/blog/kafka [...]
\ No newline at end of file
diff --git a/assets/js/29e20dbd.94a6213e.js b/assets/js/29e20dbd.94a6213e.js
new file mode 100644
index 00000000..5f283370
--- /dev/null
+++ b/assets/js/29e20dbd.94a6213e.js
@@ -0,0 +1 @@
+"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[578],{5944:(e,a,t)=>{t.r(a),t.d(a,{assets:()=>l,contentTitle:()=>s,default:()=>p,frontMatter:()=>o,metadata:()=>r,toc:()=>c});var n=t(5893),i=t(1151);const o={slug:"kafka-meets-wayang-2",title:"Apache Kafka meets Wayang - Part 2",authors:"kamir",tags:["wayang","kafka","cross organization data collaboration"]},s="Apache Wayang meets Apache Kafka - Part 2",r={permalink:"/blog/kafka-meets-wayang-2",sou [...]
\ No newline at end of file
diff --git a/assets/js/29e20dbd.f0dc9b10.js b/assets/js/29e20dbd.f0dc9b10.js
deleted file mode 100644
index d07fd7d4..00000000
--- a/assets/js/29e20dbd.f0dc9b10.js
+++ /dev/null
@@ -1 +0,0 @@
-"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[578],{5944:(e,a,t)=>{t.r(a),t.d(a,{assets:()=>l,contentTitle:()=>s,default:()=>p,frontMatter:()=>o,metadata:()=>r,toc:()=>c});var n=t(5893),i=t(1151);const o={slug:"kafka-meets-wayang-2",title:"Apache Kafka meets Apache Wayang - Part 2",authors:"kamir",tags:["wayang","kafka","cross organization data collaboration"]},s="Apache Wayang meets Apache Kafka - Part 2",r={permalink:"/blog/kafka-meets-wayang [...]
\ No newline at end of file
diff --git a/assets/js/34f284a7.4a7bccac.js b/assets/js/34f284a7.4a7bccac.js
new file mode 100644
index 00000000..162fbd89
--- /dev/null
+++ b/assets/js/34f284a7.4a7bccac.js
@@ -0,0 +1 @@
+"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[8815],{9702:(e,a,t)=>{t.r(a),t.d(a,{assets:()=>l,contentTitle:()=>s,default:()=>p,frontMatter:()=>o,metadata:()=>r,toc:()=>c});var n=t(5893),i=t(1151);const o={slug:"kafka-meets-wayang-2",title:"Apache Kafka meets Wayang - Part 2",authors:"kamir",tags:["wayang","kafka","cross organization data collaboration"]},s="Apache Wayang meets Apache Kafka - Part 2",r={permalink:"/blog/kafka-meets-wayang-2",so [...]
\ No newline at end of file
diff --git a/assets/js/34f284a7.f6acbe7e.js b/assets/js/34f284a7.f6acbe7e.js
deleted file mode 100644
index 0aa998ff..00000000
--- a/assets/js/34f284a7.f6acbe7e.js
+++ /dev/null
@@ -1 +0,0 @@
-"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[8815],{9702:(e,a,t)=>{t.r(a),t.d(a,{assets:()=>l,contentTitle:()=>s,default:()=>p,frontMatter:()=>o,metadata:()=>r,toc:()=>c});var n=t(5893),i=t(1151);const o={slug:"kafka-meets-wayang-2",title:"Apache Kafka meets Apache Wayang - Part 2",authors:"kamir",tags:["wayang","kafka","cross organization data collaboration"]},s="Apache Wayang meets Apache Kafka - Part 2",r={permalink:"/blog/kafka-meets-wayan [...]
\ No newline at end of file
diff --git a/assets/js/35a4b3d1.a1b6ca44.js b/assets/js/35a4b3d1.51b6a3ed.js
similarity index 71%
rename from assets/js/35a4b3d1.a1b6ca44.js
rename to assets/js/35a4b3d1.51b6a3ed.js
index 76f2502c..dbeea01e 100644
--- a/assets/js/35a4b3d1.a1b6ca44.js
+++ b/assets/js/35a4b3d1.51b6a3ed.js
@@ -1 +1 @@
-"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[8779],{8379:e=>{e.exports=JSON.parse('{"permalink":"/blog/tags/spark","page":1,"postsPerPage":10,"totalPages":1,"totalCount":2,"blogDescription":"Blog","blogTitle":"Blog"}')}}]);
\ No newline at end of file
+"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[8779],{8379:e=>{e.exports=JSON.parse('{"permalink":"/blog/tags/spark","page":1,"postsPerPage":10,"totalPages":1,"totalCount":1,"blogDescription":"Blog","blogTitle":"Blog"}')}}]);
\ No newline at end of file
diff --git a/assets/js/3df8af31.8b4f09b3.js b/assets/js/3df8af31.8b4f09b3.js
new file mode 100644
index 00000000..69c21780
--- /dev/null
+++ b/assets/js/3df8af31.8b4f09b3.js
@@ -0,0 +1 @@
+"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[6603],{5149:(a,e,t)=>{t.r(e),t.d(e,{assets:()=>l,contentTitle:()=>r,default:()=>p,frontMatter:()=>i,metadata:()=>s,toc:()=>c});var n=t(5893),o=t(1151);const i={slug:"kafka-meets-wayang-1",title:"Apache Kafka meets Wayang - Part 1",authors:"kamir",tags:["wayang","kafka","cross organization data collaboration"]},r="Apache Wayang meets Apache Kafka - Part 1",s={permalink:"/blog/kafka-meets-wayang-1",so [...]
\ No newline at end of file
diff --git a/assets/js/3df8af31.8c2e6b09.js b/assets/js/3df8af31.8c2e6b09.js
deleted file mode 100644
index dade2773..00000000
--- a/assets/js/3df8af31.8c2e6b09.js
+++ /dev/null
@@ -1 +0,0 @@
-"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[6603],{5149:(a,e,t)=>{t.r(e),t.d(e,{assets:()=>l,contentTitle:()=>r,default:()=>p,frontMatter:()=>i,metadata:()=>s,toc:()=>c});var n=t(5893),o=t(1151);const i={slug:"kafka-meets-wayang-1",title:"Apache Kafka meets Apache Wayang - Part 1",authors:"kamir",tags:["wayang","kafka","cross organization data collaboration"]},r="Apache Wayang meets Apache Kafka - Part 1",s={permalink:"/blog/kafka-meets-wayan [...]
\ No newline at end of file
diff --git a/assets/js/3e5cd75b.8826fd6a.js b/assets/js/3e5cd75b.8826fd6a.js
new file mode 100644
index 00000000..71352169
--- /dev/null
+++ b/assets/js/3e5cd75b.8826fd6a.js
@@ -0,0 +1 @@
+"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[8045],{6722:(a,e,t)=>{t.r(e),t.d(e,{assets:()=>l,contentTitle:()=>r,default:()=>p,frontMatter:()=>i,metadata:()=>s,toc:()=>c});var n=t(5893),o=t(1151);const i={slug:"kafka-meets-wayang-1",title:"Apache Kafka meets Wayang - Part 1",authors:"kamir",tags:["wayang","kafka","cross organization data collaboration"]},r="Apache Wayang meets Apache Kafka - Part 1",s={permalink:"/blog/kafka-meets-wayang-1",so [...]
\ No newline at end of file
diff --git a/assets/js/3e5cd75b.8b6d58e0.js b/assets/js/3e5cd75b.8b6d58e0.js
deleted file mode 100644
index f9b07b67..00000000
--- a/assets/js/3e5cd75b.8b6d58e0.js
+++ /dev/null
@@ -1 +0,0 @@
-"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[8045],{6722:(a,e,t)=>{t.r(e),t.d(e,{assets:()=>l,contentTitle:()=>r,default:()=>p,frontMatter:()=>i,metadata:()=>s,toc:()=>c});var n=t(5893),o=t(1151);const i={slug:"kafka-meets-wayang-1",title:"Apache Kafka meets Apache Wayang - Part 1",authors:"kamir",tags:["wayang","kafka","cross organization data collaboration"]},r="Apache Wayang meets Apache Kafka - Part 1",s={permalink:"/blog/kafka-meets-wayan [...]
\ No newline at end of file
diff --git a/assets/js/47457315.3fd21bbe.js b/assets/js/47457315.a462d621.js
similarity index 71%
rename from assets/js/47457315.3fd21bbe.js
rename to assets/js/47457315.a462d621.js
index 933937ea..60b4ad0e 100644
--- a/assets/js/47457315.3fd21bbe.js
+++ b/assets/js/47457315.a462d621.js
@@ -1 +1 @@
-"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[1380],{5755:e=>{e.exports=JSON.parse('{"permalink":"/blog/tags/kafka","page":1,"postsPerPage":10,"totalPages":1,"totalCount":6,"blogDescription":"Blog","blogTitle":"Blog"}')}}]);
\ No newline at end of file
+"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[1380],{5755:e=>{e.exports=JSON.parse('{"permalink":"/blog/tags/kafka","page":1,"postsPerPage":10,"totalPages":1,"totalCount":3,"blogDescription":"Blog","blogTitle":"Blog"}')}}]);
\ No newline at end of file
diff --git a/assets/js/4d06a287.3d412fc3.js b/assets/js/4d06a287.3d412fc3.js
new file mode 100644
index 00000000..22d9006d
--- /dev/null
+++ b/assets/js/4d06a287.3d412fc3.js
@@ -0,0 +1 @@
+"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[5539],{1642:(n,e,a)=>{a.r(e),a.d(e,{assets:()=>l,contentTitle:()=>s,default:()=>h,frontMatter:()=>o,metadata:()=>r,toc:()=>p});var t=a(5893),i=a(1151);const o={slug:"wayang-python-api",title:"Pywayang - Apache Wayang's Python API",authors:["juripetersen"],tags:["wayang","python"]},s="Pywayang - Apache Wayang's Python API",r={permalink:"/blog/wayang-python-api",source:"@site/blog/2024-04-09-python-ap [...]
\ No newline at end of file
diff --git a/assets/js/4d06a287.e04133cf.js b/assets/js/4d06a287.e04133cf.js
deleted file mode 100644
index 73d96c9d..00000000
--- a/assets/js/4d06a287.e04133cf.js
+++ /dev/null
@@ -1 +0,0 @@
-"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[5539],{1642:(n,e,a)=>{a.r(e),a.d(e,{assets:()=>l,contentTitle:()=>s,default:()=>h,frontMatter:()=>o,metadata:()=>r,toc:()=>p});var t=a(5893),i=a(1151);const o={slug:"wayang-python-api",title:"Pywayang - Apache Wayang's Python API",authors:["juripetersen"],tags:["wayang","python"]},s="Pywayang - Apache Wayang's Python API",r={permalink:"/blog/wayang-python-api",source:"@site/blog/2024-04-09-python-ap [...]
\ No newline at end of file
diff --git a/assets/js/5f842dbc.2b9209c3.js b/assets/js/5f842dbc.54716e90.js
similarity index 81%
rename from assets/js/5f842dbc.2b9209c3.js
rename to assets/js/5f842dbc.54716e90.js
index fe36b364..8203d288 100644
--- a/assets/js/5f842dbc.2b9209c3.js
+++ b/assets/js/5f842dbc.54716e90.js
@@ -1 +1 @@
-"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[7434],{394:a=>{a.exports=JSON.parse('{"label":"kafka","permalink":"/blog/tags/kafka","allTagsPath":"/blog/tags","count":6,"unlisted":false}')}}]);
\ No newline at end of file
+"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[7434],{394:a=>{a.exports=JSON.parse('{"label":"kafka","permalink":"/blog/tags/kafka","allTagsPath":"/blog/tags","count":3,"unlisted":false}')}}]);
\ No newline at end of file
diff --git a/assets/js/720d5f7a.af3b91c3.js b/assets/js/720d5f7a.d6da4f1e.js
similarity index 81%
rename from assets/js/720d5f7a.af3b91c3.js
rename to assets/js/720d5f7a.d6da4f1e.js
index f278a81d..b0a393d9 100644
--- a/assets/js/720d5f7a.af3b91c3.js
+++ b/assets/js/720d5f7a.d6da4f1e.js
@@ -1 +1 @@
-"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[4687],{3162:a=>{a.exports=JSON.parse('{"label":"spark","permalink":"/blog/tags/spark","allTagsPath":"/blog/tags","count":2,"unlisted":false}')}}]);
\ No newline at end of file
+"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[4687],{3162:a=>{a.exports=JSON.parse('{"label":"spark","permalink":"/blog/tags/spark","allTagsPath":"/blog/tags","count":1,"unlisted":false}')}}]);
\ No newline at end of file
diff --git a/assets/js/75f5c5f6.10d21cb5.js b/assets/js/75f5c5f6.10d21cb5.js
deleted file mode 100644
index e543e209..00000000
--- a/assets/js/75f5c5f6.10d21cb5.js
+++ /dev/null
@@ -1 +0,0 @@
-"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[3176],{1697:(e,a,t)=>{t.r(a),t.d(a,{assets:()=>l,contentTitle:()=>r,default:()=>h,frontMatter:()=>s,metadata:()=>o,toc:()=>c});var n=t(5893),i=t(1151);const s={slug:"wayang-vs-trino",title:"Apache Wayang vs. Presto/Trino",authors:["zkaoudi"],tags:["wayang","presto","trino"]},r="Apache Wayang vs. Presto/Trino",o={permalink:"/blog/wayang-vs-trino",source:"@site/blog/2024-03-08-wayang-vs-presto.md",tit [...]
\ No newline at end of file
diff --git a/assets/js/75f5c5f6.b5df1a15.js b/assets/js/75f5c5f6.b5df1a15.js
new file mode 100644
index 00000000..a0778d82
--- /dev/null
+++ b/assets/js/75f5c5f6.b5df1a15.js
@@ -0,0 +1 @@
+"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[3176],{1697:(e,a,t)=>{t.r(a),t.d(a,{assets:()=>l,contentTitle:()=>r,default:()=>h,frontMatter:()=>s,metadata:()=>o,toc:()=>c});var n=t(5893),i=t(1151);const s={slug:"wayang-vs-trino",title:"Apache Wayang vs. Presto/Trino",authors:["zkaoudi"],tags:["wayang","presto","trino"]},r="Apache Wayang vs. Presto/Trino",o={permalink:"/blog/wayang-vs-trino",source:"@site/blog/2024-03-08-wayang-vs-presto.md",tit [...]
\ No newline at end of file
diff --git a/assets/js/77806f28.e7c413b3.js b/assets/js/77806f28.6901a0cf.js
similarity index 85%
rename from assets/js/77806f28.e7c413b3.js
rename to assets/js/77806f28.6901a0cf.js
index 59060c17..cfd5050c 100644
--- a/assets/js/77806f28.e7c413b3.js
+++ b/assets/js/77806f28.6901a0cf.js
@@ -1 +1 @@
-"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[7524],{3714:a=>{a.exports=JSON.parse('{"label":"cross organization data collaboration","permalink":"/blog/tags/cross-organization-data-collaboration","allTagsPath":"/blog/tags","count":6,"unlisted":false}')}}]);
\ No newline at end of file
+"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[7524],{3714:a=>{a.exports=JSON.parse('{"label":"cross organization data collaboration","permalink":"/blog/tags/cross-organization-data-collaboration","allTagsPath":"/blog/tags","count":3,"unlisted":false}')}}]);
\ No newline at end of file
diff --git a/assets/js/7c80c937.bff51920.js b/assets/js/7c80c937.93fb8865.js
similarity index 80%
rename from assets/js/7c80c937.bff51920.js
rename to assets/js/7c80c937.93fb8865.js
index 37ec7784..6f08dcd8 100644
--- a/assets/js/7c80c937.bff51920.js
+++ b/assets/js/7c80c937.93fb8865.js
@@ -1 +1 @@
-"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[1958],{1505:a=>{a.exports=JSON.parse('{"label":"wayang","permalink":"/blog/tags/wayang","allTagsPath":"/blog/tags","count":12,"unlisted":false}')}}]);
\ No newline at end of file
+"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[1958],{1505:a=>{a.exports=JSON.parse('{"label":"wayang","permalink":"/blog/tags/wayang","allTagsPath":"/blog/tags","count":9,"unlisted":false}')}}]);
\ No newline at end of file
diff --git a/assets/js/814f3328.5ac9b2c3.js b/assets/js/814f3328.5ac9b2c3.js
deleted file mode 100644
index 6c3f1201..00000000
--- a/assets/js/814f3328.5ac9b2c3.js
+++ /dev/null
@@ -1 +0,0 @@
-"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[2535],{5641:a=>{a.exports=JSON.parse('{"title":"All our posts","items":[{"title":"Apache Kafka meets Apache Wayang : Part 1","permalink":"/blog/kafka-meets-wayang-1","unlisted":false},{"title":"Apache Kafka meets Apache Wayang - Part 2","permalink":"/blog/kafka-meets-wayang-2","unlisted":false},{"title":"Apache Kafka meets Apache Wayang - Part 3","permalink":"/blog/kafka-meets-wayang-2","unlisted":f [...]
\ No newline at end of file
diff --git a/assets/js/814f3328.f9c5cafe.js b/assets/js/814f3328.f9c5cafe.js
new file mode 100644
index 00000000..b08c6976
--- /dev/null
+++ b/assets/js/814f3328.f9c5cafe.js
@@ -0,0 +1 @@
+"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[2535],{5641:a=>{a.exports=JSON.parse('{"title":"All our posts","items":[{"title":"Apache Wayang Release Odysse","permalink":"/blog/wayang-release-odysse","unlisted":false},{"title":"Integrating ML platforms in Wayang","permalink":"/blog/wayang-tensorflow","unlisted":false},{"title":"Wayang and the Federated AI","permalink":"/blog/wayang-federated-ai","unlisted":false},{"title":"Pywayang - Apache Way [...]
\ No newline at end of file
diff --git a/assets/js/86331d88.34234831.js b/assets/js/86331d88.c1ef525c.js
similarity index 64%
rename from assets/js/86331d88.34234831.js
rename to assets/js/86331d88.c1ef525c.js
index f397ba73..8d8dba0a 100644
--- a/assets/js/86331d88.34234831.js
+++ b/assets/js/86331d88.c1ef525c.js
@@ -1 +1 @@
-"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[4266],{2313:a=>{a.exports=JSON.parse('{"permalink":"/blog/tags/cross-organization-data-collaboration","page":1,"postsPerPage":10,"totalPages":1,"totalCount":6,"blogDescription":"Blog","blogTitle":"Blog"}')}}]);
\ No newline at end of file
+"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[4266],{2313:a=>{a.exports=JSON.parse('{"permalink":"/blog/tags/cross-organization-data-collaboration","page":1,"postsPerPage":10,"totalPages":1,"totalCount":3,"blogDescription":"Blog","blogTitle":"Blog"}')}}]);
\ No newline at end of file
diff --git a/assets/js/8eb4e46b.d22aff7e.js b/assets/js/8eb4e46b.d22aff7e.js
deleted file mode 100644
index 3da4c71d..00000000
--- a/assets/js/8eb4e46b.d22aff7e.js
+++ /dev/null
@@ -1 +0,0 @@
-"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[1],{2638:e=>{e.exports=JSON.parse('{"permalink":"/blog/page/2","page":2,"postsPerPage":10,"totalPages":2,"totalCount":12,"previousPage":"/blog","blogDescription":"Blog","blogTitle":"Blog"}')}}]);
\ No newline at end of file
diff --git a/assets/js/8f973d51.053525a0.js b/assets/js/8f973d51.053525a0.js
new file mode 100644
index 00000000..d01cd89b
--- /dev/null
+++ b/assets/js/8f973d51.053525a0.js
@@ -0,0 +1 @@
+"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[4431],{4310:(e,a,n)=>{n.r(a),n.d(a,{assets:()=>l,contentTitle:()=>o,default:()=>p,frontMatter:()=>r,metadata:()=>s,toc:()=>c});var t=n(5893),i=n(1151);const r={slug:"kafka-meets-wayang-3",title:"Apache Kafka meets Wayang - Part 3",authors:"kamir",tags:["wayang","kafka","spark","cross organization data collaboration"]},o=void 0,s={permalink:"/blog/kafka-meets-wayang-3",source:"@site/blog/2024-03-10-k [...]
\ No newline at end of file
diff --git a/assets/js/8f973d51.da7f5ca3.js b/assets/js/8f973d51.da7f5ca3.js
deleted file mode 100644
index 5f174c81..00000000
--- a/assets/js/8f973d51.da7f5ca3.js
+++ /dev/null
@@ -1 +0,0 @@
-"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[4431],{4310:(e,a,n)=>{n.r(a),n.d(a,{assets:()=>l,contentTitle:()=>o,default:()=>p,frontMatter:()=>r,metadata:()=>s,toc:()=>c});var t=n(5893),i=n(1151);const r={slug:"kafka-meets-wayang-3",title:"Apache Kafka meets Apache Wayang - Part 3",authors:"kamir",tags:["wayang","kafka","spark","cross organization data collaboration"]},o=void 0,s={permalink:"/blog/kafka-meets-wayang-3",source:"@site/blog/2024- [...]
\ No newline at end of file
diff --git a/assets/js/922da4c8.2fa414fe.js b/assets/js/922da4c8.2fa414fe.js
deleted file mode 100644
index 110f31b8..00000000
--- a/assets/js/922da4c8.2fa414fe.js
+++ /dev/null
@@ -1 +0,0 @@
-"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[3224],{4335:(a,e,t)=>{t.r(e),t.d(e,{assets:()=>l,contentTitle:()=>r,default:()=>p,frontMatter:()=>i,metadata:()=>s,toc:()=>c});var n=t(5893),o=t(1151);const i={slug:"kafka-meets-wayang-1",title:"Apache Kafka meets Apache Wayang : Part 1",authors:"kamir",tags:["wayang","kafka","cross organization data collaboration"]},r="Apache Wayang meets Apache Kafka - Part 1",s={permalink:"/blog/kafka-meets-wayan [...]
\ No newline at end of file
diff --git a/assets/js/935c02bb.25b25a20.js b/assets/js/935c02bb.25b25a20.js
new file mode 100644
index 00000000..63e46164
--- /dev/null
+++ b/assets/js/935c02bb.25b25a20.js
@@ -0,0 +1 @@
+"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[2227],{3785:(e,a,n)=>{n.r(a),n.d(a,{assets:()=>l,contentTitle:()=>o,default:()=>p,frontMatter:()=>r,metadata:()=>s,toc:()=>c});var t=n(5893),i=n(1151);const r={slug:"kafka-meets-wayang-3",title:"Apache Kafka meets Wayang - Part 3",authors:"kamir",tags:["wayang","kafka","spark","cross organization data collaboration"]},o=void 0,s={permalink:"/blog/kafka-meets-wayang-3",source:"@site/blog/2024-03-10-k [...]
\ No newline at end of file
diff --git a/assets/js/935c02bb.f9ab0c3c.js b/assets/js/935c02bb.f9ab0c3c.js
deleted file mode 100644
index 4059575b..00000000
--- a/assets/js/935c02bb.f9ab0c3c.js
+++ /dev/null
@@ -1 +0,0 @@
-"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[2227],{3785:(e,a,n)=>{n.r(a),n.d(a,{assets:()=>l,contentTitle:()=>o,default:()=>p,frontMatter:()=>r,metadata:()=>s,toc:()=>c});var t=n(5893),i=n(1151);const r={slug:"kafka-meets-wayang-3",title:"Apache Kafka meets Apache Wayang - Part 3",authors:"kamir",tags:["wayang","kafka","spark","cross organization data collaboration"]},o=void 0,s={permalink:"/blog/kafka-meets-wayang-3",source:"@site/blog/2024- [...]
\ No newline at end of file
diff --git a/assets/js/a7023ddc.44152c59.js b/assets/js/a7023ddc.44152c59.js
new file mode 100644
index 00000000..cbd1f9c3
--- /dev/null
+++ b/assets/js/a7023ddc.44152c59.js
@@ -0,0 +1 @@
+"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[1713],{3457:a=>{a.exports=JSON.parse('[{"label":"wayang","permalink":"/blog/tags/wayang","count":9},{"label":"ASF","permalink":"/blog/tags/asf","count":1},{"label":"release","permalink":"/blog/tags/release","count":1},{"label":"ML","permalink":"/blog/tags/ml","count":1},{"label":"tensorflow","permalink":"/blog/tags/tensorflow","count":1},{"label":"federated","permalink":"/blog/tags/federated","count [...]
\ No newline at end of file
diff --git a/assets/js/a7023ddc.f00d0b14.js b/assets/js/a7023ddc.f00d0b14.js
deleted file mode 100644
index 1a7e5c07..00000000
--- a/assets/js/a7023ddc.f00d0b14.js
+++ /dev/null
@@ -1 +0,0 @@
-"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[1713],{3457:a=>{a.exports=JSON.parse('[{"label":"wayang","permalink":"/blog/tags/wayang","count":12},{"label":"kafka","permalink":"/blog/tags/kafka","count":6},{"label":"cross organization data collaboration","permalink":"/blog/tags/cross-organization-data-collaboration","count":6},{"label":"spark","permalink":"/blog/tags/spark","count":2},{"label":"ASF","permalink":"/blog/tags/asf","count":1},{"lab [...]
\ No newline at end of file
diff --git a/assets/js/ae568e66.7a788392.js b/assets/js/ae568e66.7a788392.js
new file mode 100644
index 00000000..5dbb6d0a
--- /dev/null
+++ b/assets/js/ae568e66.7a788392.js
@@ -0,0 +1 @@
+"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[6042],{7146:e=>{e.exports=JSON.parse('{"permalink":"/blog/tags/wayang","page":1,"postsPerPage":10,"totalPages":1,"totalCount":9,"blogDescription":"Blog","blogTitle":"Blog"}')}}]);
\ No newline at end of file
diff --git a/assets/js/ae568e66.92dd05e4.js b/assets/js/ae568e66.92dd05e4.js
deleted file mode 100644
index e756d977..00000000
--- a/assets/js/ae568e66.92dd05e4.js
+++ /dev/null
@@ -1 +0,0 @@
-"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[6042],{7146:a=>{a.exports=JSON.parse('{"permalink":"/blog/tags/wayang","page":1,"postsPerPage":10,"totalPages":2,"totalCount":12,"nextPage":"/blog/tags/wayang/page/2","blogDescription":"Blog","blogTitle":"Blog"}')}}]);
\ No newline at end of file
diff --git a/assets/js/ae720ac1.174e3afa.js b/assets/js/ae720ac1.174e3afa.js
deleted file mode 100644
index 59f1144f..00000000
--- a/assets/js/ae720ac1.174e3afa.js
+++ /dev/null
@@ -1 +0,0 @@
-"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[6977],{9283:(e,n,a)=>{a.r(n),a.d(n,{assets:()=>c,contentTitle:()=>i,default:()=>h,frontMatter:()=>o,metadata:()=>r,toc:()=>p});var t=a(5893),s=a(1151);const o={slug:"wayang-python-api",title:"Pywayang - Apache Wayang's Python API",authors:["juripetersen"],tags:["wayang","python"]},i="Pywayang - Apache Wayang's Python API",r={permalink:"/blog/wayang-python-api",source:"@site/blog/2024-04-09-python-ap [...]
\ No newline at end of file
diff --git a/assets/js/ae720ac1.e4462485.js b/assets/js/ae720ac1.e4462485.js
new file mode 100644
index 00000000..cc244088
--- /dev/null
+++ b/assets/js/ae720ac1.e4462485.js
@@ -0,0 +1 @@
+"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[6977],{9283:(e,n,a)=>{a.r(n),a.d(n,{assets:()=>c,contentTitle:()=>i,default:()=>h,frontMatter:()=>o,metadata:()=>r,toc:()=>p});var t=a(5893),s=a(1151);const o={slug:"wayang-python-api",title:"Pywayang - Apache Wayang's Python API",authors:["juripetersen"],tags:["wayang","python"]},i="Pywayang - Apache Wayang's Python API",r={permalink:"/blog/wayang-python-api",source:"@site/blog/2024-04-09-python-ap [...]
\ No newline at end of file
diff --git a/assets/js/b2b675dd.300bb8b4.js b/assets/js/b2b675dd.300bb8b4.js
new file mode 100644
index 00000000..a483e848
--- /dev/null
+++ b/assets/js/b2b675dd.300bb8b4.js
@@ -0,0 +1 @@
+"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[533],{8017:e=>{e.exports=JSON.parse('{"permalink":"/blog","page":1,"postsPerPage":10,"totalPages":1,"totalCount":9,"blogDescription":"Blog","blogTitle":"Blog"}')}}]);
\ No newline at end of file
diff --git a/assets/js/b2b675dd.b5eff2b2.js b/assets/js/b2b675dd.b5eff2b2.js
deleted file mode 100644
index 2908e4f3..00000000
--- a/assets/js/b2b675dd.b5eff2b2.js
+++ /dev/null
@@ -1 +0,0 @@
-"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[533],{8017:e=>{e.exports=JSON.parse('{"permalink":"/blog","page":1,"postsPerPage":10,"totalPages":2,"totalCount":12,"nextPage":"/blog/page/2","blogDescription":"Blog","blogTitle":"Blog"}')}}]);
\ No newline at end of file
diff --git a/assets/js/b2f554cd.389ebf9a.js b/assets/js/b2f554cd.389ebf9a.js
new file mode 100644
index 00000000..600f2bed
--- /dev/null
+++ b/assets/js/b2f554cd.389ebf9a.js
@@ -0,0 +1 @@
+"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[1477],{10:e=>{e.exports=JSON.parse('{"blogPosts":[{"id":"wayang-release-odysse","metadata":{"permalink":"/blog/wayang-release-odysse","source":"@site/blog/2024-09-01-release-odysse.md","title":"Apache Wayang Release Odysse","description":"Intro","date":"2024-09-01T00:00:00.000Z","formattedDate":"September 1, 2024","tags":[{"label":"wayang","permalink":"/blog/tags/wayang"},{"label":"ASF","permalink": [...]
\ No newline at end of file
diff --git a/assets/js/b2f554cd.50606bdc.js b/assets/js/b2f554cd.50606bdc.js
deleted file mode 100644
index 1724b696..00000000
--- a/assets/js/b2f554cd.50606bdc.js
+++ /dev/null
@@ -1 +0,0 @@
-"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[1477],{10:e=>{e.exports=JSON.parse('{"blogPosts":[{"id":"kafka-meets-wayang-1","metadata":{"permalink":"/blog/kafka-meets-wayang-1","source":"@site/blog/kafka-meets-wayang-1.md","title":"Apache Kafka meets Apache Wayang : Part 1","description":"Intro","date":"2024-09-21T07:26:48.000Z","formattedDate":"September 21, 2024","tags":[{"label":"wayang","permalink":"/blog/tags/wayang"},{"label":"kafka","pe [...]
\ No newline at end of file
diff --git a/assets/js/bae78078.70745d80.js b/assets/js/bae78078.a537a88f.js
similarity index 54%
rename from assets/js/bae78078.70745d80.js
rename to assets/js/bae78078.a537a88f.js
index c20d9ff6..de979b00 100644
--- a/assets/js/bae78078.70745d80.js
+++ b/assets/js/bae78078.a537a88f.js
@@ -1 +1 @@
-"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[4378],{6772:(e,a,t)=>{t.r(a),t.d(a,{assets:()=>c,contentTitle:()=>r,default:()=>u,frontMatter:()=>s,metadata:()=>i,toc:()=>l});var n=t(5893),o=t(1151);const s={slug:"wayang-vs-trino",title:"Apache Wayang vs. Presto/Trino",authors:["zkaoudi"],tags:["wayang","presto","trino"]},r="Apache Wayang vs. Presto/Trino",i={permalink:"/blog/wayang-vs-trino",source:"@site/blog/2024-03-08-wayang-vs-presto.md",tit [...]
\ No newline at end of file
+"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[4378],{6772:(e,a,t)=>{t.r(a),t.d(a,{assets:()=>c,contentTitle:()=>r,default:()=>u,frontMatter:()=>s,metadata:()=>i,toc:()=>l});var n=t(5893),o=t(1151);const s={slug:"wayang-vs-trino",title:"Apache Wayang vs. Presto/Trino",authors:["zkaoudi"],tags:["wayang","presto","trino"]},r="Apache Wayang vs. Presto/Trino",i={permalink:"/blog/wayang-vs-trino",source:"@site/blog/2024-03-08-wayang-vs-presto.md",tit [...]
\ No newline at end of file
diff --git a/assets/js/cfab9f0f.3319519a.js b/assets/js/cfab9f0f.3319519a.js
new file mode 100644
index 00000000..6fa26a23
--- /dev/null
+++ b/assets/js/cfab9f0f.3319519a.js
@@ -0,0 +1 @@
+"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[191],{8766:(e,a,n)=>{n.r(a),n.d(a,{assets:()=>i,contentTitle:()=>l,default:()=>h,frontMatter:()=>o,metadata:()=>c,toc:()=>s});var t=n(5893),r=n(1151);const o={slug:"wayang-release-odysse",title:"Apache Wayang Release Odysse",authors:"kamir",tags:["wayang","ASF","release"]},l="How I failed to release Apache Wayang",c={permalink:"/blog/wayang-release-odysse",source:"@site/blog/2024-09-01-release-odyss [...]
\ No newline at end of file
diff --git a/assets/js/d2f02a99.3009b741.js b/assets/js/d2f02a99.3009b741.js
deleted file mode 100644
index fb7269d1..00000000
--- a/assets/js/d2f02a99.3009b741.js
+++ /dev/null
@@ -1 +0,0 @@
-"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[4054],{8e3:e=>{e.exports=JSON.parse('{"permalink":"/blog/tags/wayang/page/2","page":2,"postsPerPage":10,"totalPages":2,"totalCount":12,"previousPage":"/blog/tags/wayang","blogDescription":"Blog","blogTitle":"Blog"}')}}]);
\ No newline at end of file
diff --git a/assets/js/de9e258c.25d98ce0.js b/assets/js/de9e258c.25d98ce0.js
deleted file mode 100644
index 3e78398e..00000000
--- a/assets/js/de9e258c.25d98ce0.js
+++ /dev/null
@@ -1 +0,0 @@
-"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[1871],{19:(e,a,t)=>{t.r(a),t.d(a,{assets:()=>l,contentTitle:()=>s,default:()=>p,frontMatter:()=>o,metadata:()=>r,toc:()=>c});var n=t(5893),i=t(1151);const o={slug:"kafka-meets-wayang-2",title:"Apache Kafka meets Apache Wayang - Part 2",authors:"kamir",tags:["wayang","kafka","cross organization data collaboration"]},s="Apache Wayang meets Apache Kafka - Part 2",r={permalink:"/blog/kafka-meets-wayang- [...]
\ No newline at end of file
diff --git a/assets/js/e82464af.d4e009f4.js b/assets/js/e82464af.efc150d0.js
similarity index 54%
rename from assets/js/e82464af.d4e009f4.js
rename to assets/js/e82464af.efc150d0.js
index 0945ae7c..bf7808c2 100644
--- a/assets/js/e82464af.d4e009f4.js
+++ b/assets/js/e82464af.efc150d0.js
@@ -1 +1 @@
-"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[7555],{1757:(e,t,a)=>{a.r(t),a.d(t,{assets:()=>i,contentTitle:()=>r,default:()=>p,frontMatter:()=>o,metadata:()=>u,toc:()=>c});var n=a(5893),s=a(1151);const o={slug:"website_update",title:"Website updated",authors:["alo.alt"],tags:["wayang"]},r=void 0,u={permalink:"/blog/website_update",source:"@site/blog/2024-01-25-website_update.md",title:"Website updated",description:"We're updated our website an [...]
\ No newline at end of file
+"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[7555],{1757:(e,t,a)=>{a.r(t),a.d(t,{assets:()=>i,contentTitle:()=>r,default:()=>p,frontMatter:()=>o,metadata:()=>u,toc:()=>c});var n=a(5893),s=a(1151);const o={slug:"website_update",title:"Website updated",authors:["alo.alt"],tags:["wayang"]},r=void 0,u={permalink:"/blog/website_update",source:"@site/blog/2024-01-25-website_update.md",title:"Website updated",description:"We're updated our website an [...]
\ No newline at end of file
diff --git a/assets/js/effa2a56.33b72a28.js b/assets/js/effa2a56.33b72a28.js
new file mode 100644
index 00000000..49b0c63e
--- /dev/null
+++ b/assets/js/effa2a56.33b72a28.js
@@ -0,0 +1 @@
+"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[3399],{897:(e,n,t)=>{t.r(n),t.d(n,{assets:()=>o,contentTitle:()=>l,default:()=>d,frontMatter:()=>i,metadata:()=>r,toc:()=>c});var s=t(5893),a=t(1151);const i={slug:"website_update",title:"Website updated",authors:["alo.alt"],tags:["wayang"]},l="Website updated",r={permalink:"/blog/website_update",source:"@site/blog/2024-01-25-website_update.md",title:"Website updated",description:"We're updated our [...]
\ No newline at end of file
diff --git a/assets/js/effa2a56.6b6490a2.js b/assets/js/effa2a56.6b6490a2.js
deleted file mode 100644
index 38454365..00000000
--- a/assets/js/effa2a56.6b6490a2.js
+++ /dev/null
@@ -1 +0,0 @@
-"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[3399],{897:(e,n,t)=>{t.r(n),t.d(n,{assets:()=>o,contentTitle:()=>l,default:()=>d,frontMatter:()=>i,metadata:()=>r,toc:()=>c});var s=t(5893),a=t(1151);const i={slug:"website_update",title:"Website updated",authors:["alo.alt"],tags:["wayang"]},l="Website updated",r={permalink:"/blog/website_update",source:"@site/blog/2024-01-25-website_update.md",title:"Website updated",description:"We're updated our [...]
\ No newline at end of file
diff --git a/assets/js/f5d9e7eb.0f4692b4.js b/assets/js/f5d9e7eb.0f4692b4.js
deleted file mode 100644
index 31f8507a..00000000
--- a/assets/js/f5d9e7eb.0f4692b4.js
+++ /dev/null
@@ -1 +0,0 @@
-"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[3967],{7512:(a,e,n)=>{n.r(e),n.d(e,{assets:()=>c,contentTitle:()=>i,default:()=>d,frontMatter:()=>r,metadata:()=>s,toc:()=>l});var t=n(5893),o=n(1151);const r={slug:"wayang-tensorflow",title:"Integrating ML platforms in Wayang",authors:["zkaoudi"],tags:["wayang","ML","tensorflow"]},i="Integrating ML platforms in Wayang",s={permalink:"/blog/wayang-tensorflow",source:"@site/blog/2024-05-07-wayang-tens [...]
\ No newline at end of file
diff --git a/assets/js/f5d9e7eb.1e9fed5e.js b/assets/js/f5d9e7eb.1e9fed5e.js
new file mode 100644
index 00000000..ca6798f5
--- /dev/null
+++ b/assets/js/f5d9e7eb.1e9fed5e.js
@@ -0,0 +1 @@
+"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[3967],{7512:(a,e,n)=>{n.r(e),n.d(e,{assets:()=>c,contentTitle:()=>i,default:()=>d,frontMatter:()=>r,metadata:()=>s,toc:()=>l});var t=n(5893),o=n(1151);const r={slug:"wayang-tensorflow",title:"Integrating ML platforms in Wayang",authors:["zkaoudi"],tags:["wayang","ML","tensorflow"]},i="Integrating ML platforms in Wayang",s={permalink:"/blog/wayang-tensorflow",source:"@site/blog/2024-05-07-wayang-tens [...]
\ No newline at end of file
diff --git a/assets/js/f9d658b5.98d642bc.js b/assets/js/f9d658b5.98d642bc.js
new file mode 100644
index 00000000..6f8ba213
--- /dev/null
+++ b/assets/js/f9d658b5.98d642bc.js
@@ -0,0 +1 @@
+"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[9298],{2687:(a,e,n)=>{n.r(e),n.d(e,{assets:()=>c,contentTitle:()=>i,default:()=>d,frontMatter:()=>r,metadata:()=>s,toc:()=>l});var t=n(5893),o=n(1151);const r={slug:"wayang-tensorflow",title:"Integrating ML platforms in Wayang",authors:["zkaoudi"],tags:["wayang","ML","tensorflow"]},i="Integrating ML platforms in Wayang",s={permalink:"/blog/wayang-tensorflow",source:"@site/blog/2024-05-07-wayang-tens [...]
\ No newline at end of file
diff --git a/assets/js/f9d658b5.f5b24665.js b/assets/js/f9d658b5.f5b24665.js
deleted file mode 100644
index 1f5a1d1a..00000000
--- a/assets/js/f9d658b5.f5b24665.js
+++ /dev/null
@@ -1 +0,0 @@
-"use strict";(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[9298],{2687:(a,e,n)=>{n.r(e),n.d(e,{assets:()=>c,contentTitle:()=>i,default:()=>d,frontMatter:()=>r,metadata:()=>s,toc:()=>l});var t=n(5893),o=n(1151);const r={slug:"wayang-tensorflow",title:"Integrating ML platforms in Wayang",authors:["zkaoudi"],tags:["wayang","ML","tensorflow"]},i="Integrating ML platforms in Wayang",s={permalink:"/blog/wayang-tensorflow",source:"@site/blog/2024-05-07-wayang-tens [...]
\ No newline at end of file
diff --git a/assets/js/main.0cc2e93a.js b/assets/js/main.0cc2e93a.js
deleted file mode 100644
index 57663d81..00000000
--- a/assets/js/main.0cc2e93a.js
+++ /dev/null
@@ -1,2 +0,0 @@
-/*! For license information please see main.0cc2e93a.js.LICENSE.txt */
-(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[179],{723:(e,t,n)=>{"use strict";n.d(t,{Z:()=>f});n(7294);var r=n(8356),a=n.n(r),o=n(6887);const i={"01a85c17":[()=>Promise.all([n.e(532),n.e(4013)]).then(n.bind(n,1223)),"@theme/BlogTagsListPage",1223],"05a94acb":[()=>n.e(3617).then(n.bind(n,7856)),"@site/docs/community/committer.md",7856],"06bc2d4d":[()=>n.e(746).then(n.bind(n,8837)),"@site/blog/2024-04-17-federated-ai.md?truncated=true",8837],"092604a4":[()=> [...]
\ No newline at end of file
diff --git a/assets/js/main.4f5176d9.js b/assets/js/main.4f5176d9.js
new file mode 100644
index 00000000..47ea0b1d
--- /dev/null
+++ b/assets/js/main.4f5176d9.js
@@ -0,0 +1,2 @@
+/*! For license information please see main.4f5176d9.js.LICENSE.txt */
+(self.webpackChunkwayang_website=self.webpackChunkwayang_website||[]).push([[179],{723:(e,t,n)=>{"use strict";n.d(t,{Z:()=>f});n(7294);var r=n(8356),a=n.n(r),o=n(6887);const i={"01a85c17":[()=>Promise.all([n.e(532),n.e(4013)]).then(n.bind(n,1223)),"@theme/BlogTagsListPage",1223],"05a94acb":[()=>n.e(3617).then(n.bind(n,7856)),"@site/docs/community/committer.md",7856],"06bc2d4d":[()=>n.e(746).then(n.bind(n,8837)),"@site/blog/2024-04-17-federated-ai.md?truncated=true",8837],"092604a4":[()=> [...]
\ No newline at end of file
diff --git a/assets/js/main.0cc2e93a.js.LICENSE.txt b/assets/js/main.4f5176d9.js.LICENSE.txt
similarity index 100%
rename from assets/js/main.0cc2e93a.js.LICENSE.txt
rename to assets/js/main.4f5176d9.js.LICENSE.txt
diff --git a/assets/js/runtime~main.01bb8940.js b/assets/js/runtime~main.01bb8940.js
deleted file mode 100644
index d83bb9f0..00000000
--- a/assets/js/runtime~main.01bb8940.js
+++ /dev/null
@@ -1 +0,0 @@
-(()=>{"use strict";var e,a,c,d,f,b={},t={};function r(e){var a=t[e];if(void 0!==a)return a.exports;var c=t[e]={exports:{}};return b[e].call(c.exports,c,c.exports,r),c.exports}r.m=b,e=[],r.O=(a,c,d,f)=>{if(!c){var b=1/0;for(i=0;i<e.length;i++){c=e[i][0],d=e[i][1],f=e[i][2];for(var t=!0,o=0;o<c.length;o++)(!1&f||b>=f)&&Object.keys(r.O).every((e=>r.O[e](c[o])))?c.splice(o--,1):(t=!1,f<b&&(b=f));if(t){e.splice(i--,1);var n=d();void 0!==n&&(a=n)}}return a}f=f||0;for(var i=e.length;i>0&&e[i-1] [...]
\ No newline at end of file
diff --git a/assets/js/runtime~main.8489d39d.js b/assets/js/runtime~main.8489d39d.js
new file mode 100644
index 00000000..e5a71c5f
--- /dev/null
+++ b/assets/js/runtime~main.8489d39d.js
@@ -0,0 +1 @@
+(()=>{"use strict";var e,a,c,d,f,b={},t={};function r(e){var a=t[e];if(void 0!==a)return a.exports;var c=t[e]={exports:{}};return b[e].call(c.exports,c,c.exports,r),c.exports}r.m=b,e=[],r.O=(a,c,d,f)=>{if(!c){var b=1/0;for(i=0;i<e.length;i++){c=e[i][0],d=e[i][1],f=e[i][2];for(var t=!0,o=0;o<c.length;o++)(!1&f||b>=f)&&Object.keys(r.O).every((e=>r.O[e](c[o])))?c.splice(o--,1):(t=!1,f<b&&(b=f));if(t){e.splice(i--,1);var n=d();void 0!==n&&(a=n)}}return a}f=f||0;for(var i=e.length;i>0&&e[i-1] [...]
\ No newline at end of file
diff --git a/blog/archive/index.html b/blog/archive/index.html
index 9d030420..45a7a6ee 100644
--- a/blog/archive/index.html
+++ b/blog/archive/index.html
@@ -5,11 +5,11 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">Archive | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/blog/archive"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docusaurus_tag" content="default"><meta data [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.01bb8940.js" defer="defer"></script>
-<script src="/assets/js/main.0cc2e93a.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.8489d39d.js" defer="defer"></script>
+<script src="/assets/js/main.4f5176d9.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
-<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
+<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
<p> Apache Wayang is an effort undergoing incubation at The Apache Software Foundation (ASF), sponsored by the Apache Incubator. Incubation is required of all newly accepted projects until a further review indicates that the infrastructure, communications, and decision making process have stabilized in a manner consistent with other successful ASF projects. While incubation status is not necessarily a reflection of the completeness or stability of the code, it does indicate that th [...]
<p>
Copyright © 2024 The Apache Software Foundation, Licensed under the Apache License, Version 2.0. <br>
diff --git a/blog/atom.xml b/blog/atom.xml
index 142083bb..e745c3b4 100644
--- a/blog/atom.xml
+++ b/blog/atom.xml
@@ -2,264 +2,138 @@
<feed xmlns="http://www.w3.org/2005/Atom">
<id>https://wayang.apache.org/blog</id>
<title>Apache Wayang (incubating) Blog</title>
- <updated>2024-09-21T07:26:48.000Z</updated>
+ <updated>2024-09-01T00:00:00.000Z</updated>
<generator>https://github.com/jpmonette/feed</generator>
<link rel="alternate" href="https://wayang.apache.org/blog"/>
<subtitle>Apache Wayang (incubating) Blog</subtitle>
<icon>https://wayang.apache.org/img/wayang-logo.jpg</icon>
<entry>
- <title type="html"><![CDATA[Apache Kafka meets Apache Wayang : Part 1]]></title>
- <id>https://wayang.apache.org/blog/kafka-meets-wayang-1</id>
- <link href="https://wayang.apache.org/blog/kafka-meets-wayang-1"/>
- <updated>2024-09-21T07:26:48.000Z</updated>
- <summary type="html"><![CDATA[Intro]]></summary>
- <content type="html"><![CDATA[<h2 class="anchor anchorWithStickyNavbar_LWe7" id="intro">Intro<a href="https://wayang.apache.org/blog/kafka-meets-wayang-1#intro" class="hash-link" aria-label="Direct link to Intro" title="Direct link to Intro"></a></h2>
-<p>This article is the first of a four part series about federated data analysis using Apache Wayang.
-The first article starts with an introduction of a typical data colaboration scenario which will emerge in our digital future.</p>
-<p>In part two and three we will share a summary of our Apache Kafka client implementation for Apache Wayang.
-We started with the Java Platform (part 2) and the Apache Spark implementation follows (W.I.P.) in part three.</p>
-<p>The use case behind this work is an imaginary data collaboration scenario.</p>
-<p>We see this example and the demand for a solution already in many places.</p>
-<p>For us this is motivation enough to propose a solution.
-This would also allow us to do more local data processing, and businesses can stop moving data around the world, but rather care about data locality while they expose and share specific information to others by using data federation.
-This reduces complexity of data management and cost dramatically.</p>
-<p>For this purpose, we illustrate a cross organizational data sharing scenario from the finance sector soon.
-This analysis pattern will also be relevant in the context of data analysis along supply chains, another typical example where data from many stakeholder together is needed but never managed in one place, for good reasons.</p>
-<p>Data federation can help us to unlock the hidden value of all those isolated data lakes.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="a-cross-organizational-data-sharing-scenario">A cross organizational data sharing scenario<a href="https://wayang.apache.org/blog/kafka-meets-wayang-1#a-cross-organizational-data-sharing-scenario" class="hash-link" aria-label="Direct link to A cross organizational data sharing scenario" title="Direct link to A cross organizational data sharing scenario"></a></h2>
-<p>Our goal is the implementation of a cross organization decentralized data processing scenario, in which protected local data should be processed in combination with public data from public sources in a collaborative manner.
-Instead of copying all data into a central data lake or a central data platform we decided to use federated analytics.
-Apache Wayang is the tool we work with.
-In our case, the public data is hosted on publicly available websites or data pods.
-A client can use the HTTP(S) protocol to read the data which is given in a well defined format.
-For simplicity we decided to use CSV format.
-When we look into the data of each participant we have a different perspective.</p>
-<p>Our processing procedure should calculate a particular metric on the <em>local data</em> of each participant.
-An example of such a metric is the average spending of all users on a particular product category per month.
-This can vary from partner to partner, hence, we want to be able to calculate a peer-group comparison so that each partner can see its own metric compared with a global average calculated from contributions by all partners.
-Such a process requires global averaging and local averaging.
-And due to governance constraints, we can’t bring all raw data together in one place.</p>
-<p>Instead, we want to use Apache Wayang for this purpose.
-We simplify the procedure and split it into two phases.
-Phase one is the process, which allows each participant to calculate the local metrics.
-This requires only local data. The second phase requires data from all collaborating partners.
-The monthly sum and counter values per partner and category are needed in one place by all other parties.
-Hence, the algorithm of the first phase stores the local results locally, and the contributions to the global results in an externally accessible Kafka topic.
-We assume this is done by each of the partners.</p>
-<p>Now we have a scenario, in which an Apache Wayang process must be able to read data from multiple Apache Kafka topics from multiple Apache Kafka clusters but finally writes into a single Kafka topic, which then can be accessed by all the participating clients.</p>
-<p><img decoding="async" loading="lazy" alt="images/image-1.png" src="https://wayang.apache.org/assets/images/image-1-9cc35d5aea2b867d7e5759a96bd02334.png" width="904" height="550" class="img_ev3q"></p>
-<p>The illustration shows the data flows in such a scenario.
-Jobs with red border are executed by the participants in isolation within their own data processing environments.
-But they share some of the data, using publicly accessible Kafka topics, marked by A. Job 4 is the Apache Wayang job in our focus: here we intent to read data from 3 different source systems, and write results into a fourth system (marked as B), which can be accesses by all participants again.</p>
-<p>With this in mind we want to implement an Apache Wayang application which implements the illustrated <em>Job 4</em>.
-Since as of today, there is now <em>KafkaSource</em> and <em>KafkaSink</em> available in Apache Wayang, an implementation of both will be our first step.
-Our assumption is, that in the beginning, there won’t be much data.</p>
-<p>Apache Spark is not required to cope with the load, but we expect, that in the future, a single Java application would not be able to handle our workload.
-Hence, we want to utilize the Apache Wayang abstraction over multiple processing platforms, starting with Java.</p>
-<p>Later, we want to switch to Apache Spark.</p>]]></content>
- <author>
- <name>Mirko Kämpf</name>
- <uri>https://github.com/kamir</uri>
- </author>
- <category label="wayang" term="wayang"/>
- <category label="kafka" term="kafka"/>
- <category label="cross organization data collaboration" term="cross organization data collaboration"/>
- </entry>
- <entry>
- <title type="html"><![CDATA[Apache Kafka meets Apache Wayang - Part 2]]></title>
- <id>https://wayang.apache.org/blog/kafka-meets-wayang-2</id>
- <link href="https://wayang.apache.org/blog/kafka-meets-wayang-2"/>
- <updated>2024-09-21T07:26:48.000Z</updated>
- <summary type="html"><![CDATA[In the second part of the article series we describe the implementation of the Kafka Source and Kafka Sink component for Apache Wayang.]]></summary>
- <content type="html"><![CDATA[<p>In the second part of the article series we describe the implementation of the Kafka Source and Kafka Sink component for Apache Wayang.
-We look into the “Read- and Write-Path” for our data items, called <em>DataQuanta</em>.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="apache-wayangs-read--write-path-for-kafka-topics">Apache Wayang’s Read & Write Path for Kafka topics<a href="https://wayang.apache.org/blog/kafka-meets-wayang-2#apache-wayangs-read--write-path-for-kafka-topics" class="hash-link" aria-label="Direct link to Apache Wayang’s Read & Write Path for Kafka topics" title="Direct link to Apache Wayang’s Read & Write Path for Kafka topics"></a></h2>
-<p>To describe the read and write paths for data in the context of the created Apache Wayang code snippet, the primary classes and interfaces we need to understand are as follows:</p>
-<p><strong>WayangContext:</strong> This class is essential for initializing the Wayang processing environment.
-It allows you to configure the execution environment and register plugins that define which platforms Wayang can use for data processing tasks, such as <em>Java.basicPlugin()</em> for local Java execution.</p>
-<p><strong>JavaPlanBuilder:</strong> This class is used to build and define the data processing pipeline (or plan) in Wayang.
-It provides a fluent API to specify the operations to be performed on the data, from reading the input to processing it and writing the output.</p>
-<h3 class="anchor anchorWithStickyNavbar_LWe7" id="read-path">Read Path<a href="https://wayang.apache.org/blog/kafka-meets-wayang-2#read-path" class="hash-link" aria-label="Direct link to Read Path" title="Direct link to Read Path"></a></h3>
-<p>The read path describes how data is ingested from a source into the Wayang processing pipeline:</p>
-<p><em>Reading from Kafka Topic:</em> The method <em>readKafkaTopic(topicName)</em> is used to ingest data from a specified Kafka topic.
-This is the starting point of the data processing pipeline, where topicName represents the name of the Kafka topic from which data is read.</p>
-<p><em>Data Tokenization and Preparation:</em> Once the data is read from Kafka, it undergoes several transformations such as Splitting, Filtering, and Mapping.
-What follows are the procedures known as Reducing, Grouping, Co-Grouping, and Counting.</p>
-<h3 class="anchor anchorWithStickyNavbar_LWe7" id="write-path">Write Path<a href="https://wayang.apache.org/blog/kafka-meets-wayang-2#write-path" class="hash-link" aria-label="Direct link to Write Path" title="Direct link to Write Path"></a></h3>
-<p><em>Writing to Kafka Topic:</em> The final step in the pipeline involves writing the processed data back to a Kafka topic using <em>.writeKafkaTopic(...)</em>.
-This method takes parameters that specify the target Kafka topic, a serialization function to format the data as strings, and additional configuration for load profile estimation, which optimizes the writing process.</p>
-<p>This read-write path provides a comprehensive flow of data from ingestion from Kafka, through various processing steps, and finally back to Kafka, showcasing a full cycle of data processing within Apache Wayang's abstracted environment and is implemented in our example program shown in <em>listing 1</em>.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="implementation-of-input--and-output-operators">Implementation of Input- and Output Operators<a href="https://wayang.apache.org/blog/kafka-meets-wayang-2#implementation-of-input--and-output-operators" class="hash-link" aria-label="Direct link to Implementation of Input- and Output Operators" title="Direct link to Implementation of Input- and Output Operators"></a></h2>
-<p>The next section shows how a new pair of operators can be implemented to extend Apache Wayang’s capabilities on the input and output side.
-We created the Kafka Source and Kafka Sink components so that our cross organizational data collaboration scenario can be implemented using data streaming infrastructure.</p>
-<p><strong>Level 1 – Wayang execution plan with abstract operators</strong></p>
-<p>The implementation of our Kafka Source and Kafka Sink components for Apache Wayang requires new methods and classes on three layers.
-First of all in the API package.
-Here we use the JavaPlanBuilder to expose the function for selecting a Kafka topic as the source to be used by client.<br>
-<!-- -->The class <em>JavaPlanBuilder</em> in package <em>org.apache.wayang.api</em> in the project <em>wayang-api/wayang-api-scala-java</em> exposes our new functionality to our external client.
-An instance of the JavaPlanBuilder is used to define the data processing pipeline.
-We use its <em>readKafkaTopic()</em> which specifies the source Kafka topic to read from, and for the write path we use the <em>writeKafkaTopic()</em> method.
-Both Methods do only trigger activities in the background.</p>
-<p>For the output side, we use the <em>DataQuantaBuilder</em> class, which offers an implementation of the writeKafkaTopic function.
-This function is designed to send processed data, referred to as DataQuanta, to a specified Kafka topic.
-Essentially, it marks the final step in a data processing sequence constructed using the Apache Wayang framework.</p>
-<p>In the DataQuanta class we implemented the methods writeKafkaTopic and writeKafkaTopicJava which use the KafkaTopicSink class.
-In this API layer we use the Scala programming language, but we utilize the Java classes, implemented in the layer below.</p>
-<p><strong>Level 2 – Wiring between Platform Abstraction and Implementation</strong></p>
-<p>The second layer builds the bridge between the WayangContext and PlanBuilders which work together with DataQuanta and the DataQuantaBuilder.</p>
-<p>Also, the mapping between the abstract components and the specific implementations are defined in this layer.</p>
-<p>Therefore, the mappings package has a class <em>Mappings</em> in which all relevant input and output operators are listed.
-We use it to register the KafkaSourceMapping and a KafkaSinkMapping for the particular platform, Java in our case.
-These classes allow the Apache Wayang framework to use the Java implementation of the KafkaTopicSource component (and KafkaTopicSink respectively).
-While the Wayang execution plan uses the higher abstractions, here on the “platform level” we have to link the specific implementation for the target platform.</p>
-<p>In our case this leads to a Java program running on a JVM which is set up by the Apache Wayang framework using the logical components of the execution plan.</p>
-<p>Those mappings link the real implementation of our operators the ones used in an execution plan.
-The JavaKafkaTopicSource and the JavaKafkaTopicSink extend the KafkaTopicSource and KafkaTopicSink so that the lower level implementation of those classes become available within Wayang’s Java Platform context.</p>
-<p>In this layer, the KafkaConsumer class and the KafkaProducer class are used, but both are configured and instantiated in the next layer underneath.
-All this is done in the project <em>wayang-plarforms/wayang-java</em>.</p>
-<p><strong>Layer 3 – Input/Output Connector Layer</strong></p>
-<p>The <em>KafkaTopicSource</em> and <em>KafkaTopicSink</em> classes build the third layer of our implementation.
-Both are implemented in Java programming language.
-In this layer, the real Kafka-Client logic is defined.
-Details about consumer and producers, client configuration, and schema handling have to be handled here.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="summary">Summary<a href="https://wayang.apache.org/blog/kafka-meets-wayang-2#summary" class="hash-link" aria-label="Direct link to Summary" title="Direct link to Summary"></a></h2>
-<p>Both classes in the third layer implement the Kafka client logic which is needed by the Wayang-execution plan when external data flows should be established.
-The layer above handles the mapping of the components at startup time.
-All this wiring is needed to keep Wayang open and flexible so that multiple external systems can be used in a variety of combinations and using multiple target platforms in combinations.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="outlook">Outlook<a href="https://wayang.apache.org/blog/kafka-meets-wayang-2#outlook" class="hash-link" aria-label="Direct link to Outlook" title="Direct link to Outlook"></a></h2>
-<p>The next part of the article series will cover the creation of an Kafka Source and Sink component for the Apache Spark platform, which allows our use case to scale.</p>
-<p>Finally, in part four we bring all puzzles together, and show the full implementation of the multi organizational data collaboration use case.</p>]]></content>
- <author>
- <name>Mirko Kämpf</name>
- <uri>https://github.com/kamir</uri>
- </author>
- <category label="wayang" term="wayang"/>
- <category label="kafka" term="kafka"/>
- <category label="cross organization data collaboration" term="cross organization data collaboration"/>
- </entry>
- <entry>
- <title type="html"><![CDATA[Apache Kafka meets Apache Wayang - Part 3]]></title>
- <id>https://wayang.apache.org/blog/kafka-meets-wayang-2</id>
- <link href="https://wayang.apache.org/blog/kafka-meets-wayang-2"/>
- <updated>2024-09-21T07:26:48.000Z</updated>
- <summary type="html"><![CDATA[* THIS ARTICLE IS WORK IN PROGRESS *]]></summary>
- <content type="html"><![CDATA[<p>In the second part of the article series we describe the implementation of the Kafka Source and Kafka Sink component for Apache Wayang.
-We look into the “Read- and Write-Path” for our data items, called <em>DataQuanta</em>.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="apache-wayangs-read--write-path-for-kafka-topics">Apache Wayang’s Read & Write Path for Kafka topics<a href="https://wayang.apache.org/blog/kafka-meets-wayang-2#apache-wayangs-read--write-path-for-kafka-topics" class="hash-link" aria-label="Direct link to Apache Wayang’s Read & Write Path for Kafka topics" title="Direct link to Apache Wayang’s Read & Write Path for Kafka topics"></a></h2>
-<p>To describe the read and write paths for data in the context of the created Apache Wayang code snippet, the primary classes and interfaces we need to understand are as follows:</p>
-<p><strong>WayangContext:</strong> This class is essential for initializing the Wayang processing environment.
-It allows you to configure the execution environment and register plugins that define which platforms Wayang can use for data processing tasks, such as <em>Java.basicPlugin()</em> for local Java execution.</p>
-<p><strong>JavaPlanBuilder:</strong> This class is used to build and define the data processing pipeline (or plan) in Wayang.
-It provides a fluent API to specify the operations to be performed on the data, from reading the input to processing it and writing the output.</p>
-<h3 class="anchor anchorWithStickyNavbar_LWe7" id="read-path">Read Path<a href="https://wayang.apache.org/blog/kafka-meets-wayang-2#read-path" class="hash-link" aria-label="Direct link to Read Path" title="Direct link to Read Path"></a></h3>
-<p>The read path describes how data is ingested from a source into the Wayang processing pipeline:</p>
-<p><em>Reading from Kafka Topic:</em> The method <em>readKafkaTopic(topicName)</em> is used to ingest data from a specified Kafka topic.
-This is the starting point of the data processing pipeline, where topicName represents the name of the Kafka topic from which data is read.</p>
-<p><em>Data Tokenization and Preparation:</em> Once the data is read from Kafka, it undergoes several transformations such as Splitting, Filtering, and Mapping.
-What follows are the procedures known as Reducing, Grouping, Co-Grouping, and Counting.</p>
-<h3 class="anchor anchorWithStickyNavbar_LWe7" id="write-path">Write Path<a href="https://wayang.apache.org/blog/kafka-meets-wayang-2#write-path" class="hash-link" aria-label="Direct link to Write Path" title="Direct link to Write Path"></a></h3>
-<p><em>Writing to Kafka Topic:</em> The final step in the pipeline involves writing the processed data back to a Kafka topic using <em>.writeKafkaTopic(...)</em>.
-This method takes parameters that specify the target Kafka topic, a serialization function to format the data as strings, and additional configuration for load profile estimation, which optimizes the writing process.</p>
-<p>This read-write path provides a comprehensive flow of data from ingestion from Kafka, through various processing steps, and finally back to Kafka, showcasing a full cycle of data processing within Apache Wayang's abstracted environment and is implemented in our example program shown in <em>listing 1</em>.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="implementation-of-input--and-output-operators">Implementation of Input- and Output Operators<a href="https://wayang.apache.org/blog/kafka-meets-wayang-2#implementation-of-input--and-output-operators" class="hash-link" aria-label="Direct link to Implementation of Input- and Output Operators" title="Direct link to Implementation of Input- and Output Operators"></a></h2>
-<p>The next section shows how a new pair of operators can be implemented to extend Apache Wayang’s capabilities on the input and output side.
-We created the Kafka Source and Kafka Sink components so that our cross organizational data collaboration scenario can be implemented using data streaming infrastructure.</p>
-<p><strong>Level 1 – Wayang execution plan with abstract operators</strong></p>
-<p>The implementation of our Kafka Source and Kafka Sink components for Apache Wayang requires new methods and classes on three layers.
-First of all in the API package.
-Here we use the JavaPlanBuilder to expose the function for selecting a Kafka topic as the source to be used by client.<br>
-<!-- -->The class <em>JavaPlanBuilder</em> in package <em>org.apache.wayang.api</em> in the project <em>wayang-api/wayang-api-scala-java</em> exposes our new functionality to our external client.
-An instance of the JavaPlanBuilder is used to define the data processing pipeline.
-We use its <em>readKafkaTopic()</em> which specifies the source Kafka topic to read from, and for the write path we use the <em>writeKafkaTopic()</em> method.
-Both Methods do only trigger activities in the background.</p>
-<p>For the output side, we use the <em>DataQuantaBuilder</em> class, which offers an implementation of the writeKafkaTopic function.
-This function is designed to send processed data, referred to as DataQuanta, to a specified Kafka topic.
-Essentially, it marks the final step in a data processing sequence constructed using the Apache Wayang framework.</p>
-<p>In the DataQuanta class we implemented the methods writeKafkaTopic and writeKafkaTopicJava which use the KafkaTopicSink class.
-In this API layer we use the Scala programming language, but we utilize the Java classes, implemented in the layer below.</p>
-<p><strong>Level 2 – Wiring between Platform Abstraction and Implementation</strong></p>
-<p>The second layer builds the bridge between the WayangContext and PlanBuilders which work together with DataQuanta and the DataQuantaBuilder.</p>
-<p>Also, the mapping between the abstract components and the specific implementations are defined in this layer.</p>
-<p>Therefore, the mappings package has a class <em>Mappings</em> in which all relevant input and output operators are listed.
-We use it to register the KafkaSourceMapping and a KafkaSinkMapping for the particular platform, Java in our case.
-These classes allow the Apache Wayang framework to use the Java implementation of the KafkaTopicSource component (and KafkaTopicSink respectively).
-While the Wayang execution plan uses the higher abstractions, here on the “platform level” we have to link the specific implementation for the target platform.</p>
-<p>In our case this leads to a Java program running on a JVM which is set up by the Apache Wayang framework using the logical components of the execution plan.</p>
-<p>Those mappings link the real implementation of our operators the ones used in an execution plan.
-The JavaKafkaTopicSource and the JavaKafkaTopicSink extend the KafkaTopicSource and KafkaTopicSink so that the lower level implementation of those classes become available within Wayang’s Java Platform context.</p>
-<p>In this layer, the KafkaConsumer class and the KafkaProducer class are used, but both are configured and instantiated in the next layer underneath.
-All this is done in the project <em>wayang-plarforms/wayang-java</em>.</p>
-<p><strong>Layer 3 – Input/Output Connector Layer</strong></p>
-<p>The <em>KafkaTopicSource</em> and <em>KafkaTopicSink</em> classes build the third layer of our implementation.
-Both are implemented in Java programming language.
-In this layer, the real Kafka-Client logic is defined.
-Details about consumer and producers, client configuration, and schema handling have to be handled here.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="summary">Summary<a href="https://wayang.apache.org/blog/kafka-meets-wayang-2#summary" class="hash-link" aria-label="Direct link to Summary" title="Direct link to Summary"></a></h2>
-<p>Both classes in the third layer implement the Kafka client logic which is needed by the Wayang-execution plan when external data flows should be established.
-The layer above handles the mapping of the components at startup time.
-All this wiring is needed to keep Wayang open and flexible so that multiple external systems can be used in a variety of combinations and using multiple target platforms in combinations.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="outlook">Outlook<a href="https://wayang.apache.org/blog/kafka-meets-wayang-2#outlook" class="hash-link" aria-label="Direct link to Outlook" title="Direct link to Outlook"></a></h2>
-<p>The next part of the article series will cover the creation of an Kafka Source and Sink component for the Apache Spark platform, which allows our use case to scale.</p>
-<p>Finally, in part four we bring all puzzles together, and show the full implementation of the multi organizational data collaboration use case.</p>]]></content>
- <author>
- <name>Mirko Kämpf</name>
- <uri>https://github.com/kamir</uri>
- </author>
- <category label="wayang" term="wayang"/>
- <category label="kafka" term="kafka"/>
- <category label="spark" term="spark"/>
- <category label="cross organization data collaboration" term="cross organization data collaboration"/>
- </entry>
- <entry>
- <title type="html"><![CDATA[Apache Kafka meets Apache Wayang - Part 1]]></title>
- <id>https://wayang.apache.org/blog/kafka-meets-wayang-1</id>
- <link href="https://wayang.apache.org/blog/kafka-meets-wayang-1"/>
- <updated>2024-09-21T07:26:48.000Z</updated>
+ <title type="html"><![CDATA[Apache Wayang Release Odysse]]></title>
+ <id>https://wayang.apache.org/blog/wayang-release-odysse</id>
+ <link href="https://wayang.apache.org/blog/wayang-release-odysse"/>
+ <updated>2024-09-01T00:00:00.000Z</updated>
<summary type="html"><![CDATA[Intro]]></summary>
- <content type="html"><![CDATA[<h2 class="anchor anchorWithStickyNavbar_LWe7" id="intro">Intro<a href="https://wayang.apache.org/blog/kafka-meets-wayang-1#intro" class="hash-link" aria-label="Direct link to Intro" title="Direct link to Intro"></a></h2>
-<p>This article is the first of a four part series about federated data analysis using Apache Wayang.
-The first article starts with an introduction of a typical data colaboration scenario which will emerge in our digital future.</p>
-<p>In part two and three we will share a summary of our Apache Kafka client implementation for Apache Wayang.
-We started with the Java Platform (part 2) and the Apache Spark implementation follows (W.I.P.) in part three.</p>
-<p>The use case behind this work is an imaginary data collaboration scenario.</p>
-<p>We see this example and the demand for a solution already in many places.</p>
-<p>For us this is motivation enough to propose a solution.
-This would also allow us to do more local data processing, and businesses can stop moving data around the world, but rather care about data locality while they expose and share specific information to others by using data federation.
-This reduces complexity of data management and cost dramatically.</p>
-<p>For this purpose, we illustrate a cross organizational data sharing scenario from the finance sector soon.
-This analysis pattern will also be relevant in the context of data analysis along supply chains, another typical example where data from many stakeholder together is needed but never managed in one place, for good reasons.</p>
-<p>Data federation can help us to unlock the hidden value of all those isolated data lakes.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="a-cross-organizational-data-sharing-scenario">A cross organizational data sharing scenario<a href="https://wayang.apache.org/blog/kafka-meets-wayang-1#a-cross-organizational-data-sharing-scenario" class="hash-link" aria-label="Direct link to A cross organizational data sharing scenario" title="Direct link to A cross organizational data sharing scenario"></a></h2>
-<p>Our goal is the implementation of a cross organization decentralized data processing scenario, in which protected local data should be processed in combination with public data from public sources in a collaborative manner.
-Instead of copying all data into a central data lake or a central data platform we decided to use federated analytics.
-Apache Wayang is the tool we work with.
-In our case, the public data is hosted on publicly available websites or data pods.
-A client can use the HTTP(S) protocol to read the data which is given in a well defined format.
-For simplicity we decided to use CSV format.
-When we look into the data of each participant we have a different perspective.</p>
-<p>Our processing procedure should calculate a particular metric on the <em>local data</em> of each participant.
-An example of such a metric is the average spending of all users on a particular product category per month.
-This can vary from partner to partner, hence, we want to be able to calculate a peer-group comparison so that each partner can see its own metric compared with a global average calculated from contributions by all partners.
-Such a process requires global averaging and local averaging.
-And due to governance constraints, we can’t bring all raw data together in one place.</p>
-<p>Instead, we want to use Apache Wayang for this purpose.
-We simplify the procedure and split it into two phases.
-Phase one is the process, which allows each participant to calculate the local metrics.
-This requires only local data. The second phase requires data from all collaborating partners.
-The monthly sum and counter values per partner and category are needed in one place by all other parties.
-Hence, the algorithm of the first phase stores the local results locally, and the contributions to the global results in an externally accessible Kafka topic.
-We assume this is done by each of the partners.</p>
-<p>Now we have a scenario, in which an Apache Wayang process must be able to read data from multiple Apache Kafka topics from multiple Apache Kafka clusters but finally writes into a single Kafka topic, which then can be accessed by all the participating clients.</p>
-<p><img decoding="async" loading="lazy" alt="images/image-1.png" src="https://wayang.apache.org/assets/images/image-1-9cc35d5aea2b867d7e5759a96bd02334.png" width="904" height="550" class="img_ev3q"></p>
-<p>The illustration shows the data flows in such a scenario.
-Jobs with red border are executed by the participants in isolation within their own data processing environments.
-But they share some of the data, using publicly accessible Kafka topics, marked by A. Job 4 is the Apache Wayang job in our focus: here we intent to read data from 3 different source systems, and write results into a fourth system (marked as B), which can be accesses by all participants again.</p>
-<p>With this in mind we want to implement an Apache Wayang application which implements the illustrated <em>Job 4</em>.
-Since as of today, there is now <em>KafkaSource</em> and <em>KafkaSink</em> available in Apache Wayang, an implementation of both will be our first step.
-Our assumption is, that in the beginning, there won’t be much data.</p>
-<p>Apache Spark is not required to cope with the load, but we expect, that in the future, a single Java application would not be able to handle our workload.
-Hence, we want to utilize the Apache Wayang abstraction over multiple processing platforms, starting with Java.</p>
-<p>Later, we want to switch to Apache Spark.</p>]]></content>
+ <content type="html"><![CDATA[<h2 class="anchor anchorWithStickyNavbar_LWe7" id="intro">Intro<a href="https://wayang.apache.org/blog/wayang-release-odysse#intro" class="hash-link" aria-label="Direct link to Intro" title="Direct link to Intro"></a></h2>
+<p>The ASF provides a robust infrastructure for open communities of software developers. We can share ideas, combine forces, contribute code, docs, review-energy, art work, and from time to time we can nail it down. A release defines an intermediate result of the continuous community work.</p>
+<p>How we do such a release in the Apache Wayang team is an essential aspect towards graduation. First of all, there are some references to take into account, such as:</p>
+<ul>
+<li><a href="https://www.apache.org/legal/release-policy.html" target="_blank" rel="noopener noreferrer">https://www.apache.org/legal/release-policy.html</a></li>
+<li><a href="https://maven.apache.org/maven-release/maven-release-plugin/" target="_blank" rel="noopener noreferrer">https://maven.apache.org/maven-release/maven-release-plugin/</a></li>
+</ul>
+<p>Assuming you are (P)PMC, and assuming that you have the right permissions for such a release, you can follow the path as described in this guide:</p>
+<ul>
+<li><a href="https://plc4x.apache.org/developers/release/release.html" target="_blank" rel="noopener noreferrer">https://plc4x.apache.org/developers/release/release.html</a></li>
+</ul>
+<p>I tried to follow exactly this procedure, several times. I failed. Here I share the current status of my <strong>release attempts</strong>.</p>
+<p>I plan a longer tour, and do not want to block the project for a long time.
+Hence I create this draft, and I hope we can unblock this project as soon as possible.</p>
+<h2 class="anchor anchorWithStickyNavbar_LWe7" id="status">Status:<a href="https://wayang.apache.org/blog/wayang-release-odysse#status" class="hash-link" aria-label="Direct link to Status:" title="Direct link to Status:"></a></h2>
+<p>I am not able to conduct the <em>mvn release:perform</em> step.
+Anything before worked, sometimes only after some digging, but it worked.</p>
+<ul>
+<li>
+<p>We assume, that due to my membership in two ASF incubator projects I am not able to upload the artefacts to the Nexus repository (H1).</p>
+</li>
+<li>
+<p>It can be, that I have not the correct user and password in my <em>settings.xml</em> file (H2).</p>
+</li>
+</ul>
+<p>But I tested a manual login to the nexus server <a href="https://repository.apache.org/service/local/staging/deploy/maven2" target="_blank" rel="noopener noreferrer">https://repository.apache.org/service/local/staging/deploy/maven2</a> with success. And beyond that I have no idea how I can verify this detail alone.</p>
+<h2 class="anchor anchorWithStickyNavbar_LWe7" id="idea--proposal">Idea / Proposal<a href="https://wayang.apache.org/blog/wayang-release-odysse#idea--proposal" class="hash-link" aria-label="Direct link to Idea / Proposal" title="Direct link to Idea / Proposal"></a></h2>
+<p>(1) It would be great, if someone - who has done a release in any other ASF project or in Apache Wayang - could follow the steps I share, so that we can check where the problem hides itself.</p>
+<p>(2) As a follow-up task, I suggest to add a <strong>Release Guide</strong> to the Apache Wayang project, including release manager onboarding steps, and checklists for the particular project, derived from the referenced sources which are listed above.</p>
+<p>But for now it is all about sharing the status (as I did serveral times on multiple chanels, including JIRA, Slack, Mailing lists) and finding a solution for Apache Wayang release 1.0.</p>
+<h2 class="anchor anchorWithStickyNavbar_LWe7" id="latest-error">Latest Error:<a href="https://wayang.apache.org/blog/wayang-release-odysse#latest-error" class="hash-link" aria-label="Direct link to Latest Error:" title="Direct link to Latest Error:"></a></h2>
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn release:perform -X -DskipTests</span><br></span></code></pre><div class="buttonGroup__atx"><button ty [...]
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">[INFO] Caused by: org.eclipse.aether.deployment.DeploymentException: Failed to deploy artifacts: Could no [...]
+<h2 class="anchor anchorWithStickyNavbar_LWe7" id="activity-log">Activity Log<a href="https://wayang.apache.org/blog/wayang-release-odysse#activity-log" class="hash-link" aria-label="Direct link to Activity Log" title="Direct link to Activity Log"></a></h2>
+<div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn release:clean</span><br></span><span class="token-line" style="color:#393A34"><span cla [...]
+<div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn release:prepare </span><span class="token parameter variable" style="color:#36acaa">-Da [...]
+<div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token punctuation" style="color:#393A34">[</span><span class="token plain">ERROR</span><span class="toke [...]
+<div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn versions:set </span><span class="token parameter variable" style="color:#36acaa">-DnewV [...]
+<div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn release:prepare </span><span class="token parameter variable" style="color:#36acaa">-Da [...]
+<div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token punctuation" style="color:#393A34">[</span><span class="token plain">ERROR</span><span class="toke [...]
+<div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token function" style="color:#d73a49">git</span><span class="token plain"> status</span><br></span><span [...]
+<div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn release:prepare </span><span class="token parameter variable" style="color:#36acaa">-Da [...]
+<div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">Caused by: org.eclipse.aether.transfer.NoRepositoryConnectorException: Blocked mirror </spa [...]
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="dependency-on-jdk-11-during-release">Dependency on JDK-11 during release<a href="https://wayang.apache.org/blog/wayang-release-odysse#dependency-on-jdk-11-during-release" class="hash-link" aria-label="Direct link to Dependency on JDK-11 during release" title="Direct link to Dependency on JDK-11 during release"></a></h3>
+<blockquote>
+<p>FIXED with local JDK11 Setup</p>
+</blockquote>
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">brew install openjdk@11</span><br></span><span class="token-line" style="color:#393A34"><span class="toke [...]
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="manual-update-of-release-version">Manual update of release-version<a href="https://wayang.apache.org/blog/wayang-release-odysse#manual-update-of-release-version" class="hash-link" aria-label="Direct link to Manual update of release-version" title="Direct link to Manual update of release-version"></a></h3>
+<p>During the release procedure, do I have to set the version here in this configuration section manually?</p>
+<div class="language-xml codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-xml codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token tag punctuation" style="color:#393A34"><</span><span class="token tag" style="color:#00009f">plug [...]
+<blockquote>
+<p>It seems that these properties must be updated manually.</p>
+</blockquote>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="warning-regarding-illegal-reflective-access-operation">Warning regarding "illegal reflective access operation"<a href="https://wayang.apache.org/blog/wayang-release-odysse#warning-regarding-illegal-reflective-access-operation" class="hash-link" aria-label="Direct link to Warning regarding "illegal reflective access operation"" title="Direct link to Warning regarding "illegal reflective access operation""></a></h3>
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">[ERROR] WARNING: An illegal reflective access operation has occurred</span><br></span><span class="token- [...]
+<blockquote>
+<p><strong>This is still an OPEN ISSUE !</strong></p>
+</blockquote>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="rat-check-fails">RAT Check fails<a href="https://wayang.apache.org/blog/wayang-release-odysse#rat-check-fails" class="hash-link" aria-label="Direct link to RAT Check fails" title="Direct link to RAT Check fails"></a></h3>
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">[INFO] [ERROR] Failed to execute goal org.apache.rat:apache-rat-plugin:0.13:check (license-check) on proj [...]
+<div class="language-cat codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-cat codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#393A34" [...]
+<blockquote>
+<p><em>FIXED by adding .java-versions to .gitignore</em></p>
+</blockquote>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="tag-could-not-be-created-in-scm">Tag could not be created in SCM.<a href="https://wayang.apache.org/blog/wayang-release-odysse#tag-could-not-be-created-in-scm" class="hash-link" aria-label="Direct link to Tag could not be created in SCM." title="Direct link to Tag could not be created in SCM."></a></h3>
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">[ERROR] Failed to execute goal org.apache.maven.plugins:maven-release-plugin:3.0.1:prepare (default-cli) [...]
+<blockquote>
+<p>FIXED by manual changes in pom.xml.</p>
+</blockquote>
+<div class="language-xml codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-xml codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#393A34" [...]
+<div class="language-xml codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-xml codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token tag punctuation" style="color:#393A34"><</span><span class="token tag" style="color:#00009f">conf [...]
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn release:prepare -Darguments='-DskipTests=True -Dresume=True'</span><br></span></code></pre><div class [...]
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn clean package</span><br></span><span class="token-line" style="color:#393A34"><span class="token plai [...]
+<blockquote>
+<p>So far so good. But now the sun went down.</p>
+</blockquote>
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">[INFO] [ERROR] Failed to execute goal org.apache.maven.plugins:maven-deploy-plugin:3.0.0-M1:deploy (defau [...]
+<p>The null pointer exception indicates problems in <em>settings.xml</em>. This could be fixed.
+But now, the error changes with and I am still not able to stage the build results.</p>
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn release:perform -X -DskipTests</span><br></span></code></pre><div class="buttonGroup__atx"><button ty [...]
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">[INFO] Caused by: org.eclipse.aether.deployment.DeploymentException: Failed to deploy artifacts: Could no [...]
+<h2 class="anchor anchorWithStickyNavbar_LWe7" id="checklist-for-next-iteration">Checklist for next iteration:<a href="https://wayang.apache.org/blog/wayang-release-odysse#checklist-for-next-iteration" class="hash-link" aria-label="Direct link to Checklist for next iteration:" title="Direct link to Checklist for next iteration:"></a></h2>
+<p>The error you're encountering indicates that there is a problem with deploying the artifacts using the <code>maven-deploy-plugin</code>. The root cause of the error is a <code>NullPointerException</code> during the deployment process, specifically related to the <code>org.eclipse.aether.transfer.ArtifactTransferException</code>.</p>
+<p>Here's a step-by-step guide to troubleshoot and resolve this issue:</p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="1-check-maven-settings">1. Check Maven Settings<a href="https://wayang.apache.org/blog/wayang-release-odysse#1-check-maven-settings" class="hash-link" aria-label="Direct link to 1. Check Maven Settings" title="Direct link to 1. Check Maven Settings"></a></h3>
+<p>Ensure that your Maven settings (<code>settings.xml</code>) are correctly configured for deployment. Verify that the repository settings and credentials are correctly specified.</p>
+<p><strong>[DONE]</strong></p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="2-verify-repository-url">2. Verify Repository URL<a href="https://wayang.apache.org/blog/wayang-release-odysse#2-verify-repository-url" class="hash-link" aria-label="Direct link to 2. Verify Repository URL" title="Direct link to 2. Verify Repository URL"></a></h3>
+<p>Make sure the repository URL in your <code>pom.xml</code> or <code>settings.xml</code> is correct and reachable. The URL should point to the correct staging repository for deployment.</p>
+<p><em>[OPEN]</em> - I did not touch it, so I guess it is correct in the <em>pom.xml</em>.</p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="3-maven-version-compatibility">3. Maven Version Compatibility<a href="https://wayang.apache.org/blog/wayang-release-odysse#3-maven-version-compatibility" class="hash-link" aria-label="Direct link to 3. Maven Version Compatibility" title="Direct link to 3. Maven Version Compatibility"></a></h3>
+<p>Verify that you are using a compatible version of Maven. Sometimes, upgrading or downgrading Maven can resolve such issues.</p>
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">Apache Maven 3.9.6 (bc0240f3c744dd6b6ec2920b3cd08dcc295161ae)</span><br></span><span class="token-line" s [...]
+<p><strong>[DONE]</strong></p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="4-check-for-network-issues">4. Check for Network Issues<a href="https://wayang.apache.org/blog/wayang-release-odysse#4-check-for-network-issues" class="hash-link" aria-label="Direct link to 4. Check for Network Issues" title="Direct link to 4. Check for Network Issues"></a></h3>
+<p>Ensure there are no network issues that might be causing problems in connecting to the repository. Sometimes, network configurations, firewalls, or proxy settings can interfere with the deployment process.</p>
+<p><strong>[DONE]</strong></p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="5-update-maven-plugins">5. Update Maven Plugins<a href="https://wayang.apache.org/blog/wayang-release-odysse#5-update-maven-plugins" class="hash-link" aria-label="Direct link to 5. Update Maven Plugins" title="Direct link to 5. Update Maven Plugins"></a></h3>
+<p>Ensure that you are using the latest versions of the Maven plugins. Sometimes, bugs in older versions can cause unexpected issues.</p>
+<p><em>[OPEN]</em></p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="6-configure-the-maven-deploy-plugin-in-pomxml">6. Configure the <code>maven-deploy-plugin</code> in <code>pom.xml</code><a href="https://wayang.apache.org/blog/wayang-release-odysse#6-configure-the-maven-deploy-plugin-in-pomxml" class="hash-link" aria-label="Direct link to 6-configure-the-maven-deploy-plugin-in-pomxml" title="Direct link to 6-configure-the-maven-deploy-plugin-in-pomxml"></a></h3>
+<p>Make sure the <code>maven-deploy-plugin</code> is correctly configured in your <code>pom.xml</code>. Here's an example configuration:</p>
+<div class="language-xml codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-xml codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token tag punctuation" style="color:#393A34"><</span><span class="token tag" style="color:#00009f">plug [...]
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="7-increase-verbose-logging">7. Increase Verbose Logging<a href="https://wayang.apache.org/blog/wayang-release-odysse#7-increase-verbose-logging" class="hash-link" aria-label="Direct link to 7. Increase Verbose Logging" title="Direct link to 7. Increase Verbose Logging"></a></h3>
+<p>Enable verbose logging to get more details about the error. You can do this by adding the <code>-X</code> flag when running the Maven command:</p>
+<div class="language-sh codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-sh codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn clean deploy </span><span class="token parameter variable" style="color:#36acaa">-X</span>< [...]
+<p><strong>[DONE]</strong></p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="8-retry-with-a-clean-local-repository">8. Retry with a Clean Local Repository<a href="https://wayang.apache.org/blog/wayang-release-odysse#8-retry-with-a-clean-local-repository" class="hash-link" aria-label="Direct link to 8. Retry with a Clean Local Repository" title="Direct link to 8. Retry with a Clean Local Repository"></a></h3>
+<p>Sometimes, a corrupt local repository can cause issues. Try cleaning your local Maven repository and re-running the deployment:</p>
+<div class="language-sh codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-sh codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn clean </span><span class="token function" style="color:#d73a49">install</span><span class=" [...]
+<p><em>[OPEN]</em> - Does not match to an authentication issue.</p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="9-check-for-missing-credentials">9. Check for Missing Credentials<a href="https://wayang.apache.org/blog/wayang-release-odysse#9-check-for-missing-credentials" class="hash-link" aria-label="Direct link to 9. Check for Missing Credentials" title="Direct link to 9. Check for Missing Credentials"></a></h3>
+<p>Ensure that the credentials for the repository are correctly set up in your <code>settings.xml</code>:</p>
+<div class="language-xml codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-xml codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token tag punctuation" style="color:#393A34"><</span><span class="token tag" style="color:#00009f">serv [...]
+<p><strong>[DONE]</strong></p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="10-review-the-full-stack-trace">10. Review the Full Stack Trace<a href="https://wayang.apache.org/blog/wayang-release-odysse#10-review-the-full-stack-trace" class="hash-link" aria-label="Direct link to 10. Review the Full Stack Trace" title="Direct link to 10. Review the Full Stack Trace"></a></h3>
+<p>The full stack trace indicates a <code>NullPointerException</code>:</p>
+<div class="language-plaintext codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-plaintext codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">Caused by: java.lang.NullPointerException</span><br></span><span class="token-lin [...]
+<p>This suggests that there might be an issue with how credentials are being handled. Double-check that the credentials are being correctly passed and processed.</p>
+<p><strong>[DONE]</strong></p>]]></content>
<author>
<name>Mirko Kämpf</name>
<uri>https://github.com/kamir</uri>
@@ -480,7 +354,7 @@ expect to see soon.</p>
<category label="python" term="python"/>
</entry>
<entry>
- <title type="html"><![CDATA[Apache Kafka meets Apache Wayang - Part 3]]></title>
+ <title type="html"><![CDATA[Apache Kafka meets Wayang - Part 3]]></title>
<id>https://wayang.apache.org/blog/kafka-meets-wayang-3</id>
<link href="https://wayang.apache.org/blog/kafka-meets-wayang-3"/>
<updated>2024-03-10T00:00:00.000Z</updated>
@@ -614,7 +488,7 @@ In fact, Trino can be easily plugged to Wayang as a platform and be seamlessly i
<category label="trino" term="trino"/>
</entry>
<entry>
- <title type="html"><![CDATA[Apache Kafka meets Apache Wayang - Part 2]]></title>
+ <title type="html"><![CDATA[Apache Kafka meets Wayang - Part 2]]></title>
<id>https://wayang.apache.org/blog/kafka-meets-wayang-2</id>
<link href="https://wayang.apache.org/blog/kafka-meets-wayang-2"/>
<updated>2024-03-06T00:00:00.000Z</updated>
@@ -643,8 +517,8 @@ We created the Kafka Source and Kafka Sink components so that our cross organiza
<p><strong>Level 1 – Wayang execution plan with abstract operators</strong></p>
<p>The implementation of our Kafka Source and Kafka Sink components for Apache Wayang requires new methods and classes on three layers.
First of all in the API package.
-Here we use the JavaPlanBuilder to expose the function for selecting a Kafka topic as the source to be used by client.<br>
-<!-- -->The class <em>JavaPlanBuilder</em> in package <em>org.apache.wayang.api</em> in the project <em>wayang-api/wayang-api-scala-java</em> exposes our new functionality to our external client.
+Here we use the JavaPlanBuilder to expose the function for selecting a Kafka topic as the source to be used by client.
+The class <em>JavaPlanBuilder</em> in package <em>org.apache.wayang.api</em> in the project <em>wayang-api/wayang-api-scala-java</em> exposes our new functionality to our external client.
An instance of the JavaPlanBuilder is used to define the data processing pipeline.
We use its <em>readKafkaTopic()</em> which specifies the source Kafka topic to read from, and for the write path we use the <em>writeKafkaTopic()</em> method.
Both Methods do only trigger activities in the background.</p>
@@ -659,8 +533,8 @@ In this API layer we use the Scala programming language, but we utilize the Java
<p>Therefore, the mappings package has a class <em>Mappings</em> in which all relevant input and output operators are listed.
We use it to register the KafkaSourceMapping and a KafkaSinkMapping for the particular platform, Java in our case.
These classes allow the Apache Wayang framework to use the Java implementation of the KafkaTopicSource component (and KafkaTopicSink respectively).
-While the Wayang execution plan uses the higher abstractions, here on the “platform level” we have to link the specific implementation for the target platform.</p>
-<p>In our case this leads to a Java program running on a JVM which is set up by the Apache Wayang framework using the logical components of the execution plan.</p>
+While the Wayang execution plan uses the higher abstractions, here on the “platform level” we have to link the specific implementation for the target platform.
+In our case this leads to a Java program running on a JVM which is set up by the Apache Wayang framework using the logical components of the execution plan.</p>
<p>Those mappings link the real implementation of our operators the ones used in an execution plan.
The JavaKafkaTopicSource and the JavaKafkaTopicSink extend the KafkaTopicSource and KafkaTopicSink so that the lower level implementation of those classes become available within Wayang’s Java Platform context.</p>
<p>In this layer, the KafkaConsumer class and the KafkaProducer class are used, but both are configured and instantiated in the next layer underneath.
@@ -675,8 +549,8 @@ Details about consumer and producers, client configuration, and schema handling
The layer above handles the mapping of the components at startup time.
All this wiring is needed to keep Wayang open and flexible so that multiple external systems can be used in a variety of combinations and using multiple target platforms in combinations.</p>
<h2 class="anchor anchorWithStickyNavbar_LWe7" id="outlook">Outlook<a href="https://wayang.apache.org/blog/kafka-meets-wayang-2#outlook" class="hash-link" aria-label="Direct link to Outlook" title="Direct link to Outlook"></a></h2>
-<p>The next part of the article series will cover the creation of an Kafka Source and Sink component for the Apache Spark platform, which allows our use case to scale.</p>
-<p>Finally, in part four we bring all puzzles together, and show the full implementation of the multi organizational data collaboration use case.</p>]]></content>
+<p>The next part of the article series will cover the creation of an Kafka Source and Sink component for the Apache Spark platform, which allows our use case to scale.
+Finally, in part four we bring all puzzles together, and show the full implementation of the multi organizational data collaboration use case.</p>]]></content>
<author>
<name>Mirko Kämpf</name>
<uri>https://github.com/kamir</uri>
@@ -686,7 +560,7 @@ All this wiring is needed to keep Wayang open and flexible so that multiple exte
<category label="cross organization data collaboration" term="cross organization data collaboration"/>
</entry>
<entry>
- <title type="html"><![CDATA[Apache Kafka meets Apache Wayang - Part 1]]></title>
+ <title type="html"><![CDATA[Apache Kafka meets Wayang - Part 1]]></title>
<id>https://wayang.apache.org/blog/kafka-meets-wayang-1</id>
<link href="https://wayang.apache.org/blog/kafka-meets-wayang-1"/>
<updated>2024-03-05T00:00:00.000Z</updated>
@@ -696,9 +570,9 @@ All this wiring is needed to keep Wayang open and flexible so that multiple exte
The first article starts with an introduction of a typical data colaboration scenario which will emerge in our digital future.</p>
<p>In part two and three we will share a summary of our Apache Kafka client implementation for Apache Wayang.
We started with the Java Platform (part 2) and the Apache Spark implementation follows (W.I.P.) in part three.</p>
-<p>The use case behind this work is an imaginary data collaboration scenario.</p>
-<p>We see this example and the demand for a solution already in many places.</p>
-<p>For us this is motivation enough to propose a solution.
+<p>The use case behind this work is an imaginary data collaboration scenario.
+We see this example and the demand for a solution already in many places.
+For us this is motivation enough to propose a solution.
This would also allow us to do more local data processing, and businesses can stop moving data around the world, but rather care about data locality while they expose and share specific information to others by using data federation.
This reduces complexity of data management and cost dramatically.</p>
<p>For this purpose, we illustrate a cross organizational data sharing scenario from the finance sector soon.
@@ -733,8 +607,8 @@ But they share some of the data, using publicly accessible Kafka topics, marked
Since as of today, there is now <em>KafkaSource</em> and <em>KafkaSink</em> available in Apache Wayang, an implementation of both will be our first step.
Our assumption is, that in the beginning, there won’t be much data.</p>
<p>Apache Spark is not required to cope with the load, but we expect, that in the future, a single Java application would not be able to handle our workload.
-Hence, we want to utilize the Apache Wayang abstraction over multiple processing platforms, starting with Java.</p>
-<p>Later, we want to switch to Apache Spark.</p>]]></content>
+Hence, we want to utilize the Apache Wayang abstraction over multiple processing platforms, starting with Java.
+Later, we want to switch to Apache Spark.</p>]]></content>
<author>
<name>Mirko Kämpf</name>
<uri>https://github.com/kamir</uri>
diff --git a/blog/index.html b/blog/index.html
index e3df3619..68d499e1 100644
--- a/blog/index.html
+++ b/blog/index.html
@@ -5,211 +5,131 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">Blog | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/blog"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language" content="en"><meta data-rh="true" p [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.01bb8940.js" defer="defer"></script>
-<script src="/assets/js/main.0cc2e93a.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.8489d39d.js" defer="defer"></script>
+<script src="/assets/js/main.4f5176d9.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
-<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
-<p>This article is the first of a four part series about federated data analysis using Apache Wayang.
-The first article starts with an introduction of a typical data colaboration scenario which will emerge in our digital future.</p>
-<p>In part two and three we will share a summary of our Apache Kafka client implementation for Apache Wayang.
-We started with the Java Platform (part 2) and the Apache Spark implementation follows (W.I.P.) in part three.</p>
-<p>The use case behind this work is an imaginary data collaboration scenario.
-We see this example and the demand for a solution already in many places.
-For us this is motivation enough to propose a solution.
-This would also allow us to do more local data processing, and businesses can stop moving data around the world, but rather care about data locality while they expose and share specific information to others by using data federation.
-This reduces complexity of data management and cost dramatically.</p>
-<p>For this purpose, we illustrate a cross organizational data sharing scenario from the finance sector soon.
-This analysis pattern will also be relevant in the context of data analysis along supply chains, another typical example where data from many stakeholder together is needed but never managed in one place, for good reasons.</p>
-<p>Data federation can help us to unlock the hidden value of all those isolated data lakes.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="a-cross-organizational-data-sharing-scenario">A cross organizational data sharing scenario<a href="#a-cross-organizational-data-sharing-scenario" class="hash-link" aria-label="Direct link to A cross organizational data sharing scenario" title="Direct link to A cross organizational data sharing scenario"></a></h2>
-<p>Our goal is the implementation of a cross organization decentralized data processing scenario, in which protected local data should be processed in combination with public data from public sources in a collaborative manner.
-Instead of copying all data into a central data lake or a central data platform we decided to use federated analytics.
-Apache Wayang is the tool we work with.
-In our case, the public data is hosted on publicly available websites or data pods.
-A client can use the HTTP(S) protocol to read the data which is given in a well defined format.
-For simplicity we decided to use CSV format.
-When we look into the data of each participant we have a different perspective.</p>
-<p>Our processing procedure should calculate a particular metric on the <em>local data</em> of each participant.
-An example of such a metric is the average spending of all users on a particular product category per month.
-This can vary from partner to partner, hence, we want to be able to calculate a peer-group comparison so that each partner can see its own metric compared with a global average calculated from contributions by all partners.
-Such a process requires global averaging and local averaging.
-And due to governance constraints, we can’t bring all raw data together in one place.</p>
-<p>Instead, we want to use Apache Wayang for this purpose.
-We simplify the procedure and split it into two phases.
-Phase one is the process, which allows each participant to calculate the local metrics.
-This requires only local data. The second phase requires data from all collaborating partners.
-The monthly sum and counter values per partner and category are needed in one place by all other parties.
-Hence, the algorithm of the first phase stores the local results locally, and the contributions to the global results in an externally accessible Kafka topic.
-We assume this is done by each of the partners.</p>
-<p>Now we have a scenario, in which an Apache Wayang process must be able to read data from multiple Apache Kafka topics from multiple Apache Kafka clusters but finally writes into a single Kafka topic, which then can be accessed by all the participating clients.</p>
-<p><img decoding="async" loading="lazy" alt="images/image-1.png" src="/assets/images/image-1-9cc35d5aea2b867d7e5759a96bd02334.png" width="904" height="550" class="img_ev3q"></p>
-<p>The illustration shows the data flows in such a scenario.
-Jobs with red border are executed by the participants in isolation within their own data processing environments.
-But they share some of the data, using publicly accessible Kafka topics, marked by A. Job 4 is the Apache Wayang job in our focus: here we intent to read data from 3 different source systems, and write results into a fourth system (marked as B), which can be accesses by all participants again.</p>
-<p>With this in mind we want to implement an Apache Wayang application which implements the illustrated <em>Job 4</em>.
-Since as of today, there is now <em>KafkaSource</em> and <em>KafkaSink</em> available in Apache Wayang, an implementation of both will be our first step.
-Our assumption is, that in the beginning, there won’t be much data.</p>
-<p>Apache Spark is not required to cope with the load, but we expect, that in the future, a single Java application would not be able to handle our workload.
-Hence, we want to utilize the Apache Wayang abstraction over multiple processing platforms, starting with Java.
-Later, we want to switch to Apache Spark.</p></div><footer class="row docusaurus-mt-lg"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/kafka">kafka</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/cross-organization-data-collaboration">cross organization [...]
-We look into the “Read- and Write-Path” for our data items, called <em>DataQuanta</em>.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="apache-wayangs-read--write-path-for-kafka-topics">Apache Wayang’s Read & Write Path for Kafka topics<a href="#apache-wayangs-read--write-path-for-kafka-topics" class="hash-link" aria-label="Direct link to Apache Wayang’s Read & Write Path for Kafka topics" title="Direct link to Apache Wayang’s Read & Write Path for Kafka topics"></a></h2>
-<p>To describe the read and write paths for data in the context of the created Apache Wayang code snippet, the primary classes and interfaces we need to understand are as follows:</p>
-<p><strong>WayangContext:</strong> This class is essential for initializing the Wayang processing environment.
-It allows you to configure the execution environment and register plugins that define which platforms Wayang can use for data processing tasks, such as <em>Java.basicPlugin()</em> for local Java execution.</p>
-<p><strong>JavaPlanBuilder:</strong> This class is used to build and define the data processing pipeline (or plan) in Wayang.
-It provides a fluent API to specify the operations to be performed on the data, from reading the input to processing it and writing the output.</p>
-<h3 class="anchor anchorWithStickyNavbar_LWe7" id="read-path">Read Path<a href="#read-path" class="hash-link" aria-label="Direct link to Read Path" title="Direct link to Read Path"></a></h3>
-<p>The read path describes how data is ingested from a source into the Wayang processing pipeline:</p>
-<p><em>Reading from Kafka Topic:</em> The method <em>readKafkaTopic(topicName)</em> is used to ingest data from a specified Kafka topic.
-This is the starting point of the data processing pipeline, where topicName represents the name of the Kafka topic from which data is read.</p>
-<p><em>Data Tokenization and Preparation:</em> Once the data is read from Kafka, it undergoes several transformations such as Splitting, Filtering, and Mapping.
-What follows are the procedures known as Reducing, Grouping, Co-Grouping, and Counting.</p>
-<h3 class="anchor anchorWithStickyNavbar_LWe7" id="write-path">Write Path<a href="#write-path" class="hash-link" aria-label="Direct link to Write Path" title="Direct link to Write Path"></a></h3>
-<p><em>Writing to Kafka Topic:</em> The final step in the pipeline involves writing the processed data back to a Kafka topic using <em>.writeKafkaTopic(...)</em>.
-This method takes parameters that specify the target Kafka topic, a serialization function to format the data as strings, and additional configuration for load profile estimation, which optimizes the writing process.</p>
-<p>This read-write path provides a comprehensive flow of data from ingestion from Kafka, through various processing steps, and finally back to Kafka, showcasing a full cycle of data processing within Apache Wayang's abstracted environment and is implemented in our example program shown in <em>listing 1</em>.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="implementation-of-input--and-output-operators">Implementation of Input- and Output Operators<a href="#implementation-of-input--and-output-operators" class="hash-link" aria-label="Direct link to Implementation of Input- and Output Operators" title="Direct link to Implementation of Input- and Output Operators"></a></h2>
-<p>The next section shows how a new pair of operators can be implemented to extend Apache Wayang’s capabilities on the input and output side.
-We created the Kafka Source and Kafka Sink components so that our cross organizational data collaboration scenario can be implemented using data streaming infrastructure.</p>
-<p><strong>Level 1 – Wayang execution plan with abstract operators</strong></p>
-<p>The implementation of our Kafka Source and Kafka Sink components for Apache Wayang requires new methods and classes on three layers.
-First of all in the API package.
-Here we use the JavaPlanBuilder to expose the function for selecting a Kafka topic as the source to be used by client.
-The class <em>JavaPlanBuilder</em> in package <em>org.apache.wayang.api</em> in the project <em>wayang-api/wayang-api-scala-java</em> exposes our new functionality to our external client.
-An instance of the JavaPlanBuilder is used to define the data processing pipeline.
-We use its <em>readKafkaTopic()</em> which specifies the source Kafka topic to read from, and for the write path we use the <em>writeKafkaTopic()</em> method.
-Both Methods do only trigger activities in the background.</p>
-<p>For the output side, we use the <em>DataQuantaBuilder</em> class, which offers an implementation of the writeKafkaTopic function.
-This function is designed to send processed data, referred to as DataQuanta, to a specified Kafka topic.
-Essentially, it marks the final step in a data processing sequence constructed using the Apache Wayang framework.</p>
-<p>In the DataQuanta class we implemented the methods writeKafkaTopic and writeKafkaTopicJava which use the KafkaTopicSink class.
-In this API layer we use the Scala programming language, but we utilize the Java classes, implemented in the layer below.</p>
-<p><strong>Level 2 – Wiring between Platform Abstraction and Implementation</strong></p>
-<p>The second layer builds the bridge between the WayangContext and PlanBuilders which work together with DataQuanta and the DataQuantaBuilder.</p>
-<p>Also, the mapping between the abstract components and the specific implementations are defined in this layer.</p>
-<p>Therefore, the mappings package has a class <em>Mappings</em> in which all relevant input and output operators are listed.
-We use it to register the KafkaSourceMapping and a KafkaSinkMapping for the particular platform, Java in our case.
-These classes allow the Apache Wayang framework to use the Java implementation of the KafkaTopicSource component (and KafkaTopicSink respectively).
-While the Wayang execution plan uses the higher abstractions, here on the “platform level” we have to link the specific implementation for the target platform.
-In our case this leads to a Java program running on a JVM which is set up by the Apache Wayang framework using the logical components of the execution plan.</p>
-<p>Those mappings link the real implementation of our operators the ones used in an execution plan.
-The JavaKafkaTopicSource and the JavaKafkaTopicSink extend the KafkaTopicSource and KafkaTopicSink so that the lower level implementation of those classes become available within Wayang’s Java Platform context.</p>
-<p>In this layer, the KafkaConsumer class and the KafkaProducer class are used, but both are configured and instantiated in the next layer underneath.
-All this is done in the project <em>wayang-plarforms/wayang-java</em>.</p>
-<p><strong>Layer 3 – Input/Output Connector Layer</strong></p>
-<p>The <em>KafkaTopicSource</em> and <em>KafkaTopicSink</em> classes build the third layer of our implementation.
-Both are implemented in Java programming language.
-In this layer, the real Kafka-Client logic is defined.
-Details about consumer and producers, client configuration, and schema handling have to be handled here.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="summary">Summary<a href="#summary" class="hash-link" aria-label="Direct link to Summary" title="Direct link to Summary"></a></h2>
-<p>Both classes in the third layer implement the Kafka client logic which is needed by the Wayang-execution plan when external data flows should be established.
-The layer above handles the mapping of the components at startup time.
-All this wiring is needed to keep Wayang open and flexible so that multiple external systems can be used in a variety of combinations and using multiple target platforms in combinations.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="outlook">Outlook<a href="#outlook" class="hash-link" aria-label="Direct link to Outlook" title="Direct link to Outlook"></a></h2>
-<p>The next part of the article series will cover the creation of an Kafka Source and Sink component for the Apache Spark platform, which allows our use case to scale.
-Finally, in part four we bring all puzzles together, and show the full implementation of the multi organizational data collaboration use case.</p></div><footer class="row docusaurus-mt-lg"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/kafka">kafka</a></li><li class="tag_QGVx"><a class= [...]
-We look into the “Read- and Write-Path” for our data items, called <em>DataQuanta</em>.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="apache-wayangs-read--write-path-for-kafka-topics">Apache Wayang’s Read & Write Path for Kafka topics<a href="#apache-wayangs-read--write-path-for-kafka-topics" class="hash-link" aria-label="Direct link to Apache Wayang’s Read & Write Path for Kafka topics" title="Direct link to Apache Wayang’s Read & Write Path for Kafka topics"></a></h2>
-<p>To describe the read and write paths for data in the context of the created Apache Wayang code snippet, the primary classes and interfaces we need to understand are as follows:</p>
-<p><strong>WayangContext:</strong> This class is essential for initializing the Wayang processing environment.
-It allows you to configure the execution environment and register plugins that define which platforms Wayang can use for data processing tasks, such as <em>Java.basicPlugin()</em> for local Java execution.</p>
-<p><strong>JavaPlanBuilder:</strong> This class is used to build and define the data processing pipeline (or plan) in Wayang.
-It provides a fluent API to specify the operations to be performed on the data, from reading the input to processing it and writing the output.</p>
-<h3 class="anchor anchorWithStickyNavbar_LWe7" id="read-path">Read Path<a href="#read-path" class="hash-link" aria-label="Direct link to Read Path" title="Direct link to Read Path"></a></h3>
-<p>The read path describes how data is ingested from a source into the Wayang processing pipeline:</p>
-<p><em>Reading from Kafka Topic:</em> The method <em>readKafkaTopic(topicName)</em> is used to ingest data from a specified Kafka topic.
-This is the starting point of the data processing pipeline, where topicName represents the name of the Kafka topic from which data is read.</p>
-<p><em>Data Tokenization and Preparation:</em> Once the data is read from Kafka, it undergoes several transformations such as Splitting, Filtering, and Mapping.
-What follows are the procedures known as Reducing, Grouping, Co-Grouping, and Counting.</p>
-<h3 class="anchor anchorWithStickyNavbar_LWe7" id="write-path">Write Path<a href="#write-path" class="hash-link" aria-label="Direct link to Write Path" title="Direct link to Write Path"></a></h3>
-<p><em>Writing to Kafka Topic:</em> The final step in the pipeline involves writing the processed data back to a Kafka topic using <em>.writeKafkaTopic(...)</em>.
-This method takes parameters that specify the target Kafka topic, a serialization function to format the data as strings, and additional configuration for load profile estimation, which optimizes the writing process.</p>
-<p>This read-write path provides a comprehensive flow of data from ingestion from Kafka, through various processing steps, and finally back to Kafka, showcasing a full cycle of data processing within Apache Wayang's abstracted environment and is implemented in our example program shown in <em>listing 1</em>.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="implementation-of-input--and-output-operators">Implementation of Input- and Output Operators<a href="#implementation-of-input--and-output-operators" class="hash-link" aria-label="Direct link to Implementation of Input- and Output Operators" title="Direct link to Implementation of Input- and Output Operators"></a></h2>
-<p>The next section shows how a new pair of operators can be implemented to extend Apache Wayang’s capabilities on the input and output side.
-We created the Kafka Source and Kafka Sink components so that our cross organizational data collaboration scenario can be implemented using data streaming infrastructure.</p>
-<p><strong>Level 1 – Wayang execution plan with abstract operators</strong></p>
-<p>The implementation of our Kafka Source and Kafka Sink components for Apache Wayang requires new methods and classes on three layers.
-First of all in the API package.
-Here we use the JavaPlanBuilder to expose the function for selecting a Kafka topic as the source to be used by client.
-The class <em>JavaPlanBuilder</em> in package <em>org.apache.wayang.api</em> in the project <em>wayang-api/wayang-api-scala-java</em> exposes our new functionality to our external client.
-An instance of the JavaPlanBuilder is used to define the data processing pipeline.
-We use its <em>readKafkaTopic()</em> which specifies the source Kafka topic to read from, and for the write path we use the <em>writeKafkaTopic()</em> method.
-Both Methods do only trigger activities in the background.</p>
-<p>For the output side, we use the <em>DataQuantaBuilder</em> class, which offers an implementation of the writeKafkaTopic function.
-This function is designed to send processed data, referred to as DataQuanta, to a specified Kafka topic.
-Essentially, it marks the final step in a data processing sequence constructed using the Apache Wayang framework.</p>
-<p>In the DataQuanta class we implemented the methods writeKafkaTopic and writeKafkaTopicJava which use the KafkaTopicSink class.
-In this API layer we use the Scala programming language, but we utilize the Java classes, implemented in the layer below.</p>
-<p><strong>Level 2 – Wiring between Platform Abstraction and Implementation</strong></p>
-<p>The second layer builds the bridge between the WayangContext and PlanBuilders which work together with DataQuanta and the DataQuantaBuilder.</p>
-<p>Also, the mapping between the abstract components and the specific implementations are defined in this layer.</p>
-<p>Therefore, the mappings package has a class <em>Mappings</em> in which all relevant input and output operators are listed.
-We use it to register the KafkaSourceMapping and a KafkaSinkMapping for the particular platform, Java in our case.
-These classes allow the Apache Wayang framework to use the Java implementation of the KafkaTopicSource component (and KafkaTopicSink respectively).
-While the Wayang execution plan uses the higher abstractions, here on the “platform level” we have to link the specific implementation for the target platform.
-In our case this leads to a Java program running on a JVM which is set up by the Apache Wayang framework using the logical components of the execution plan.</p>
-<p>Those mappings link the real implementation of our operators the ones used in an execution plan.
-The JavaKafkaTopicSource and the JavaKafkaTopicSink extend the KafkaTopicSource and KafkaTopicSink so that the lower level implementation of those classes become available within Wayang’s Java Platform context.</p>
-<p>In this layer, the KafkaConsumer class and the KafkaProducer class are used, but both are configured and instantiated in the next layer underneath.
-All this is done in the project <em>wayang-plarforms/wayang-java</em>.</p>
-<p><strong>Layer 3 – Input/Output Connector Layer</strong></p>
-<p>The <em>KafkaTopicSource</em> and <em>KafkaTopicSink</em> classes build the third layer of our implementation.
-Both are implemented in Java programming language.
-In this layer, the real Kafka-Client logic is defined.
-Details about consumer and producers, client configuration, and schema handling have to be handled here.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="summary">Summary<a href="#summary" class="hash-link" aria-label="Direct link to Summary" title="Direct link to Summary"></a></h2>
-<p>Both classes in the third layer implement the Kafka client logic which is needed by the Wayang-execution plan when external data flows should be established.
-The layer above handles the mapping of the components at startup time.
-All this wiring is needed to keep Wayang open and flexible so that multiple external systems can be used in a variety of combinations and using multiple target platforms in combinations.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="outlook">Outlook<a href="#outlook" class="hash-link" aria-label="Direct link to Outlook" title="Direct link to Outlook"></a></h2>
-<p>The next part of the article series will cover the creation of an Kafka Source and Sink component for the Apache Spark platform, which allows our use case to scale.
-Finally, in part four we bring all puzzles together, and show the full implementation of the multi organizational data collaboration use case.</p></div><footer class="row docusaurus-mt-lg"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/kafka">kafka</a></li><li class="tag_QGVx"><a class= [...]
-<p>This article is the first of a four part series about federated data analysis using Apache Wayang.
-The first article starts with an introduction of a typical data colaboration scenario which will emerge in our digital future.</p>
-<p>In part two and three we will share a summary of our Apache Kafka client implementation for Apache Wayang.
-We started with the Java Platform (part 2) and the Apache Spark implementation follows (W.I.P.) in part three.</p>
-<p>The use case behind this work is an imaginary data collaboration scenario.
-We see this example and the demand for a solution already in many places.
-For us this is motivation enough to propose a solution.
-This would also allow us to do more local data processing, and businesses can stop moving data around the world, but rather care about data locality while they expose and share specific information to others by using data federation.
-This reduces complexity of data management and cost dramatically.</p>
-<p>For this purpose, we illustrate a cross organizational data sharing scenario from the finance sector soon.
-This analysis pattern will also be relevant in the context of data analysis along supply chains, another typical example where data from many stakeholder together is needed but never managed in one place, for good reasons.</p>
-<p>Data federation can help us to unlock the hidden value of all those isolated data lakes.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="a-cross-organizational-data-sharing-scenario">A cross organizational data sharing scenario<a href="#a-cross-organizational-data-sharing-scenario" class="hash-link" aria-label="Direct link to A cross organizational data sharing scenario" title="Direct link to A cross organizational data sharing scenario"></a></h2>
-<p>Our goal is the implementation of a cross organization decentralized data processing scenario, in which protected local data should be processed in combination with public data from public sources in a collaborative manner.
-Instead of copying all data into a central data lake or a central data platform we decided to use federated analytics.
-Apache Wayang is the tool we work with.
-In our case, the public data is hosted on publicly available websites or data pods.
-A client can use the HTTP(S) protocol to read the data which is given in a well defined format.
-For simplicity we decided to use CSV format.
-When we look into the data of each participant we have a different perspective.</p>
-<p>Our processing procedure should calculate a particular metric on the <em>local data</em> of each participant.
-An example of such a metric is the average spending of all users on a particular product category per month.
-This can vary from partner to partner, hence, we want to be able to calculate a peer-group comparison so that each partner can see its own metric compared with a global average calculated from contributions by all partners.
-Such a process requires global averaging and local averaging.
-And due to governance constraints, we can’t bring all raw data together in one place.</p>
-<p>Instead, we want to use Apache Wayang for this purpose.
-We simplify the procedure and split it into two phases.
-Phase one is the process, which allows each participant to calculate the local metrics.
-This requires only local data. The second phase requires data from all collaborating partners.
-The monthly sum and counter values per partner and category are needed in one place by all other parties.
-Hence, the algorithm of the first phase stores the local results locally, and the contributions to the global results in an externally accessible Kafka topic.
-We assume this is done by each of the partners.</p>
-<p>Now we have a scenario, in which an Apache Wayang process must be able to read data from multiple Apache Kafka topics from multiple Apache Kafka clusters but finally writes into a single Kafka topic, which then can be accessed by all the participating clients.</p>
-<p><img decoding="async" loading="lazy" alt="images/image-1.png" src="/assets/images/image-1-9cc35d5aea2b867d7e5759a96bd02334.png" width="904" height="550" class="img_ev3q"></p>
-<p>The illustration shows the data flows in such a scenario.
-Jobs with red border are executed by the participants in isolation within their own data processing environments.
-But they share some of the data, using publicly accessible Kafka topics, marked by A. Job 4 is the Apache Wayang job in our focus: here we intent to read data from 3 different source systems, and write results into a fourth system (marked as B), which can be accesses by all participants again.</p>
-<p>With this in mind we want to implement an Apache Wayang application which implements the illustrated <em>Job 4</em>.
-Since as of today, there is now <em>KafkaSource</em> and <em>KafkaSink</em> available in Apache Wayang, an implementation of both will be our first step.
-Our assumption is, that in the beginning, there won’t be much data.</p>
-<p>Apache Spark is not required to cope with the load, but we expect, that in the future, a single Java application would not be able to handle our workload.
-Hence, we want to utilize the Apache Wayang abstraction over multiple processing platforms, starting with Java.
-Later, we want to switch to Apache Spark.</p></div><footer class="row docusaurus-mt-lg"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/kafka">kafka</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/cross-organization-data-collaboration">cross organization [...]
+<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
+<p>The ASF provides a robust infrastructure for open communities of software developers. We can share ideas, combine forces, contribute code, docs, review-energy, art work, and from time to time we can nail it down. A release defines an intermediate result of the continuous community work.</p>
+<p>How we do such a release in the Apache Wayang team is an essential aspect towards graduation. First of all, there are some references to take into account, such as:</p>
+<ul>
+<li><a href="https://www.apache.org/legal/release-policy.html" target="_blank" rel="noopener noreferrer">https://www.apache.org/legal/release-policy.html</a></li>
+<li><a href="https://maven.apache.org/maven-release/maven-release-plugin/" target="_blank" rel="noopener noreferrer">https://maven.apache.org/maven-release/maven-release-plugin/</a></li>
+</ul>
+<p>Assuming you are (P)PMC, and assuming that you have the right permissions for such a release, you can follow the path as described in this guide:</p>
+<ul>
+<li><a href="https://plc4x.apache.org/developers/release/release.html" target="_blank" rel="noopener noreferrer">https://plc4x.apache.org/developers/release/release.html</a></li>
+</ul>
+<p>I tried to follow exactly this procedure, several times. I failed. Here I share the current status of my <strong>release attempts</strong>.</p>
+<p>I plan a longer tour, and do not want to block the project for a long time.
+Hence I create this draft, and I hope we can unblock this project as soon as possible.</p>
+<h2 class="anchor anchorWithStickyNavbar_LWe7" id="status">Status:<a href="#status" class="hash-link" aria-label="Direct link to Status:" title="Direct link to Status:"></a></h2>
+<p>I am not able to conduct the <em>mvn release:perform</em> step.
+Anything before worked, sometimes only after some digging, but it worked.</p>
+<ul>
+<li>
+<p>We assume, that due to my membership in two ASF incubator projects I am not able to upload the artefacts to the Nexus repository (H1).</p>
+</li>
+<li>
+<p>It can be, that I have not the correct user and password in my <em>settings.xml</em> file (H2).</p>
+</li>
+</ul>
+<p>But I tested a manual login to the nexus server <a href="https://repository.apache.org/service/local/staging/deploy/maven2" target="_blank" rel="noopener noreferrer">https://repository.apache.org/service/local/staging/deploy/maven2</a> with success. And beyond that I have no idea how I can verify this detail alone.</p>
+<h2 class="anchor anchorWithStickyNavbar_LWe7" id="idea--proposal">Idea / Proposal<a href="#idea--proposal" class="hash-link" aria-label="Direct link to Idea / Proposal" title="Direct link to Idea / Proposal"></a></h2>
+<p>(1) It would be great, if someone - who has done a release in any other ASF project or in Apache Wayang - could follow the steps I share, so that we can check where the problem hides itself.</p>
+<p>(2) As a follow-up task, I suggest to add a <strong>Release Guide</strong> to the Apache Wayang project, including release manager onboarding steps, and checklists for the particular project, derived from the referenced sources which are listed above.</p>
+<p>But for now it is all about sharing the status (as I did serveral times on multiple chanels, including JIRA, Slack, Mailing lists) and finding a solution for Apache Wayang release 1.0.</p>
+<h2 class="anchor anchorWithStickyNavbar_LWe7" id="latest-error">Latest Error:<a href="#latest-error" class="hash-link" aria-label="Direct link to Latest Error:" title="Direct link to Latest Error:"></a></h2>
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn release:perform -X -DskipTests</span><br></span></code></pre><div class="buttonGroup__atx"><button ty [...]
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">[INFO] Caused by: org.eclipse.aether.deployment.DeploymentException: Failed to deploy artifacts: Could no [...]
+<h2 class="anchor anchorWithStickyNavbar_LWe7" id="activity-log">Activity Log<a href="#activity-log" class="hash-link" aria-label="Direct link to Activity Log" title="Direct link to Activity Log"></a></h2>
+<div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn release:clean</span><br></span><span class="token-line" style="color:#393A34"><span cla [...]
+<div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn release:prepare </span><span class="token parameter variable" style="color:#36acaa">-Da [...]
+<div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token punctuation" style="color:#393A34">[</span><span class="token plain">ERROR</span><span class="toke [...]
+<div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn versions:set </span><span class="token parameter variable" style="color:#36acaa">-DnewV [...]
+<div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn release:prepare </span><span class="token parameter variable" style="color:#36acaa">-Da [...]
+<div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token punctuation" style="color:#393A34">[</span><span class="token plain">ERROR</span><span class="toke [...]
+<div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token function" style="color:#d73a49">git</span><span class="token plain"> status</span><br></span><span [...]
+<div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn release:prepare </span><span class="token parameter variable" style="color:#36acaa">-Da [...]
+<div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">Caused by: org.eclipse.aether.transfer.NoRepositoryConnectorException: Blocked mirror </spa [...]
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="dependency-on-jdk-11-during-release">Dependency on JDK-11 during release<a href="#dependency-on-jdk-11-during-release" class="hash-link" aria-label="Direct link to Dependency on JDK-11 during release" title="Direct link to Dependency on JDK-11 during release"></a></h3>
+<blockquote>
+<p>FIXED with local JDK11 Setup</p>
+</blockquote>
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">brew install openjdk@11</span><br></span><span class="token-line" style="color:#393A34"><span class="toke [...]
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="manual-update-of-release-version">Manual update of release-version<a href="#manual-update-of-release-version" class="hash-link" aria-label="Direct link to Manual update of release-version" title="Direct link to Manual update of release-version"></a></h3>
+<p>During the release procedure, do I have to set the version here in this configuration section manually?</p>
+<div class="language-xml codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-xml codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token tag punctuation" style="color:#393A34"><</span><span class="token tag" style="color:#00009f">plug [...]
+<blockquote>
+<p>It seems that these properties must be updated manually.</p>
+</blockquote>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="warning-regarding-illegal-reflective-access-operation">Warning regarding "illegal reflective access operation"<a href="#warning-regarding-illegal-reflective-access-operation" class="hash-link" aria-label="Direct link to Warning regarding "illegal reflective access operation"" title="Direct link to Warning regarding "illegal reflective access operation""></a></h3>
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">[ERROR] WARNING: An illegal reflective access operation has occurred</span><br></span><span class="token- [...]
+<blockquote>
+<p><strong>This is still an OPEN ISSUE !</strong></p>
+</blockquote>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="rat-check-fails">RAT Check fails<a href="#rat-check-fails" class="hash-link" aria-label="Direct link to RAT Check fails" title="Direct link to RAT Check fails"></a></h3>
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">[INFO] [ERROR] Failed to execute goal org.apache.rat:apache-rat-plugin:0.13:check (license-check) on proj [...]
+<div class="language-cat codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-cat codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#393A34" [...]
+<blockquote>
+<p><em>FIXED by adding .java-versions to .gitignore</em></p>
+</blockquote>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="tag-could-not-be-created-in-scm">Tag could not be created in SCM.<a href="#tag-could-not-be-created-in-scm" class="hash-link" aria-label="Direct link to Tag could not be created in SCM." title="Direct link to Tag could not be created in SCM."></a></h3>
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">[ERROR] Failed to execute goal org.apache.maven.plugins:maven-release-plugin:3.0.1:prepare (default-cli) [...]
+<blockquote>
+<p>FIXED by manual changes in pom.xml.</p>
+</blockquote>
+<div class="language-xml codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-xml codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#393A34" [...]
+<div class="language-xml codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-xml codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token tag punctuation" style="color:#393A34"><</span><span class="token tag" style="color:#00009f">conf [...]
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn release:prepare -Darguments='-DskipTests=True -Dresume=True'</span><br></span></code></pre> [...]
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn clean package</span><br></span><span class="token-line" style="color:#393A34"><span class="token plai [...]
+<blockquote>
+<p>So far so good. But now the sun went down.</p>
+</blockquote>
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">[INFO] [ERROR] Failed to execute goal org.apache.maven.plugins:maven-deploy-plugin:3.0.0-M1:deploy (defau [...]
+<p>The null pointer exception indicates problems in <em>settings.xml</em>. This could be fixed.
+But now, the error changes with and I am still not able to stage the build results.</p>
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn release:perform -X -DskipTests</span><br></span></code></pre><div class="buttonGroup__atx"><button ty [...]
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">[INFO] Caused by: org.eclipse.aether.deployment.DeploymentException: Failed to deploy artifacts: Could no [...]
+<h2 class="anchor anchorWithStickyNavbar_LWe7" id="checklist-for-next-iteration">Checklist for next iteration:<a href="#checklist-for-next-iteration" class="hash-link" aria-label="Direct link to Checklist for next iteration:" title="Direct link to Checklist for next iteration:"></a></h2>
+<p>The error you're encountering indicates that there is a problem with deploying the artifacts using the <code>maven-deploy-plugin</code>. The root cause of the error is a <code>NullPointerException</code> during the deployment process, specifically related to the <code>org.eclipse.aether.transfer.ArtifactTransferException</code>.</p>
+<p>Here's a step-by-step guide to troubleshoot and resolve this issue:</p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="1-check-maven-settings">1. Check Maven Settings<a href="#1-check-maven-settings" class="hash-link" aria-label="Direct link to 1. Check Maven Settings" title="Direct link to 1. Check Maven Settings"></a></h3>
+<p>Ensure that your Maven settings (<code>settings.xml</code>) are correctly configured for deployment. Verify that the repository settings and credentials are correctly specified.</p>
+<p><strong>[DONE]</strong></p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="2-verify-repository-url">2. Verify Repository URL<a href="#2-verify-repository-url" class="hash-link" aria-label="Direct link to 2. Verify Repository URL" title="Direct link to 2. Verify Repository URL"></a></h3>
+<p>Make sure the repository URL in your <code>pom.xml</code> or <code>settings.xml</code> is correct and reachable. The URL should point to the correct staging repository for deployment.</p>
+<p><em>[OPEN]</em> - I did not touch it, so I guess it is correct in the <em>pom.xml</em>.</p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="3-maven-version-compatibility">3. Maven Version Compatibility<a href="#3-maven-version-compatibility" class="hash-link" aria-label="Direct link to 3. Maven Version Compatibility" title="Direct link to 3. Maven Version Compatibility"></a></h3>
+<p>Verify that you are using a compatible version of Maven. Sometimes, upgrading or downgrading Maven can resolve such issues.</p>
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">Apache Maven 3.9.6 (bc0240f3c744dd6b6ec2920b3cd08dcc295161ae)</span><br></span><span class="token-line" s [...]
+<p><strong>[DONE]</strong></p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="4-check-for-network-issues">4. Check for Network Issues<a href="#4-check-for-network-issues" class="hash-link" aria-label="Direct link to 4. Check for Network Issues" title="Direct link to 4. Check for Network Issues"></a></h3>
+<p>Ensure there are no network issues that might be causing problems in connecting to the repository. Sometimes, network configurations, firewalls, or proxy settings can interfere with the deployment process.</p>
+<p><strong>[DONE]</strong></p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="5-update-maven-plugins">5. Update Maven Plugins<a href="#5-update-maven-plugins" class="hash-link" aria-label="Direct link to 5. Update Maven Plugins" title="Direct link to 5. Update Maven Plugins"></a></h3>
+<p>Ensure that you are using the latest versions of the Maven plugins. Sometimes, bugs in older versions can cause unexpected issues.</p>
+<p><em>[OPEN]</em></p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="6-configure-the-maven-deploy-plugin-in-pomxml">6. Configure the <code>maven-deploy-plugin</code> in <code>pom.xml</code><a href="#6-configure-the-maven-deploy-plugin-in-pomxml" class="hash-link" aria-label="Direct link to 6-configure-the-maven-deploy-plugin-in-pomxml" title="Direct link to 6-configure-the-maven-deploy-plugin-in-pomxml"></a></h3>
+<p>Make sure the <code>maven-deploy-plugin</code> is correctly configured in your <code>pom.xml</code>. Here's an example configuration:</p>
+<div class="language-xml codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-xml codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token tag punctuation" style="color:#393A34"><</span><span class="token tag" style="color:#00009f">plug [...]
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="7-increase-verbose-logging">7. Increase Verbose Logging<a href="#7-increase-verbose-logging" class="hash-link" aria-label="Direct link to 7. Increase Verbose Logging" title="Direct link to 7. Increase Verbose Logging"></a></h3>
+<p>Enable verbose logging to get more details about the error. You can do this by adding the <code>-X</code> flag when running the Maven command:</p>
+<div class="language-sh codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-sh codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn clean deploy </span><span class="token parameter variable" style="color:#36acaa">-X</span>< [...]
+<p><strong>[DONE]</strong></p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="8-retry-with-a-clean-local-repository">8. Retry with a Clean Local Repository<a href="#8-retry-with-a-clean-local-repository" class="hash-link" aria-label="Direct link to 8. Retry with a Clean Local Repository" title="Direct link to 8. Retry with a Clean Local Repository"></a></h3>
+<p>Sometimes, a corrupt local repository can cause issues. Try cleaning your local Maven repository and re-running the deployment:</p>
+<div class="language-sh codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-sh codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn clean </span><span class="token function" style="color:#d73a49">install</span><span class=" [...]
+<p><em>[OPEN]</em> - Does not match to an authentication issue.</p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="9-check-for-missing-credentials">9. Check for Missing Credentials<a href="#9-check-for-missing-credentials" class="hash-link" aria-label="Direct link to 9. Check for Missing Credentials" title="Direct link to 9. Check for Missing Credentials"></a></h3>
+<p>Ensure that the credentials for the repository are correctly set up in your <code>settings.xml</code>:</p>
+<div class="language-xml codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-xml codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token tag punctuation" style="color:#393A34"><</span><span class="token tag" style="color:#00009f">serv [...]
+<p><strong>[DONE]</strong></p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="10-review-the-full-stack-trace">10. Review the Full Stack Trace<a href="#10-review-the-full-stack-trace" class="hash-link" aria-label="Direct link to 10. Review the Full Stack Trace" title="Direct link to 10. Review the Full Stack Trace"></a></h3>
+<p>The full stack trace indicates a <code>NullPointerException</code>:</p>
+<div class="language-plaintext codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-plaintext codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">Caused by: java.lang.NullPointerException</span><br></span><span class="token-lin [...]
+<p>This suggests that there might be an issue with how credentials are being handled. Double-check that the credentials are being correctly passed and processed.</p>
+<p><strong>[DONE]</strong></p></div><footer class="row docusaurus-mt-lg"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/asf">ASF</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/release">release</a></li></ul></div></footer></article><article class="margi [...]
Thanks to the extensible nature of Wayang, the only core changes we had to do were introducing the concept of a <code>Model</code> and implement a new driver for the newly added platform.</p>
<h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-1-introducing-a-model">Step 1: Introducing a Model<a href="#step-1-introducing-a-model" class="hash-link" aria-label="Direct link to Step 1: Introducing a Model" title="Direct link to Step 1: Introducing a Model"></a></h2>
<p>With respect to the model, we followed Wayang’s abstraction philosophy: We created a <code>Model</code> interface to be used as input or output by Wayang operators and then extended it for the platform-specific operators. Different model interfaces can be found here:</p>
@@ -346,7 +266,7 @@ be executed on iterables in their respective language.
After that, the API can be added as a module in Wayang, so that
operators will be wrapped and UDFs can be executed in the desired
programming language.</li>
-</ul></div><footer class="row docusaurus-mt-lg"><div class="col col--9"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/python">python</a></li></ul></div><div class="col text--right col--3"><a aria-label="Read more about Pywayang - Apache Wayang's Python API" href="/blog/wayang-python-api"><b>Read M [...]
+</ul></div><footer class="row docusaurus-mt-lg"><div class="col col--9"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/python">python</a></li></ul></div><div class="col text--right col--3"><a aria-label="Read more about Pywayang - Apache Wayang's Python API" href="/blog/wayang-python-api"><b>Read M [...]
Motivated by the learnings from last time, I stated implementing a Kafka Source component and a Kafka Sink component for the Apache Spark platform in Apache Wayang.
In our previous article we shared the results of the work on the frist Apache Kafka integration using the Java Platform.</p>
<p>Let's see how it goes this time with Apache Spark.</p>
@@ -414,7 +334,7 @@ More testing, more serialization schemes, and Kafka Schema Registry support shou
<h2 class="anchor anchorWithStickyNavbar_LWe7" id="outlook">Outlook<a href="#outlook" class="hash-link" aria-label="Direct link to Outlook" title="Direct link to Outlook"></a></h2>
<p>The next part of the article series will cover the real world example as described in image 1.
We will show how analysts and developers can use the Apache Kafka integration for Apache Wayang to solve cross organizational collaboration issues.
-Therefore, we will bring all puzzles together, and show the full implementation of the multi organizational data collaboration use case.</p></div><footer class="row docusaurus-mt-lg"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/kafka">kafka</a></li><li class="tag_QGVx"><a class="tag_z [...]
+Therefore, we will bring all puzzles together, and show the full implementation of the multi organizational data collaboration use case.</p></div><footer class="row docusaurus-mt-lg"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/kafka">kafka</a></li><li class="tag_QGVx"><a class="tag_z [...]
We look into the “Read- and Write-Path” for our data items, called <em>DataQuanta</em>.</p>
<h2 class="anchor anchorWithStickyNavbar_LWe7" id="apache-wayangs-read--write-path-for-kafka-topics">Apache Wayang’s Read & Write Path for Kafka topics<a href="#apache-wayangs-read--write-path-for-kafka-topics" class="hash-link" aria-label="Direct link to Apache Wayang’s Read & Write Path for Kafka topics" title="Direct link to Apache Wayang’s Read & Write Path for Kafka topics"></a></h2>
<p>To describe the read and write paths for data in the context of the created Apache Wayang code snippet, the primary classes and interfaces we need to understand are as follows:</p>
@@ -471,7 +391,50 @@ The layer above handles the mapping of the components at startup time.
All this wiring is needed to keep Wayang open and flexible so that multiple external systems can be used in a variety of combinations and using multiple target platforms in combinations.</p>
<h2 class="anchor anchorWithStickyNavbar_LWe7" id="outlook">Outlook<a href="#outlook" class="hash-link" aria-label="Direct link to Outlook" title="Direct link to Outlook"></a></h2>
<p>The next part of the article series will cover the creation of an Kafka Source and Sink component for the Apache Spark platform, which allows our use case to scale.
-Finally, in part four we bring all puzzles together, and show the full implementation of the multi organizational data collaboration use case.</p></div><footer class="row docusaurus-mt-lg"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/kafka">kafka</a></li><li class="tag_QGVx"><a class= [...]
+Finally, in part four we bring all puzzles together, and show the full implementation of the multi organizational data collaboration use case.</p></div><footer class="row docusaurus-mt-lg"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/kafka">kafka</a></li><li class="tag_QGVx"><a class= [...]
+<p>This article is the first of a four part series about federated data analysis using Apache Wayang.
+The first article starts with an introduction of a typical data colaboration scenario which will emerge in our digital future.</p>
+<p>In part two and three we will share a summary of our Apache Kafka client implementation for Apache Wayang.
+We started with the Java Platform (part 2) and the Apache Spark implementation follows (W.I.P.) in part three.</p>
+<p>The use case behind this work is an imaginary data collaboration scenario.
+We see this example and the demand for a solution already in many places.
+For us this is motivation enough to propose a solution.
+This would also allow us to do more local data processing, and businesses can stop moving data around the world, but rather care about data locality while they expose and share specific information to others by using data federation.
+This reduces complexity of data management and cost dramatically.</p>
+<p>For this purpose, we illustrate a cross organizational data sharing scenario from the finance sector soon.
+This analysis pattern will also be relevant in the context of data analysis along supply chains, another typical example where data from many stakeholder together is needed but never managed in one place, for good reasons.</p>
+<p>Data federation can help us to unlock the hidden value of all those isolated data lakes.</p>
+<h2 class="anchor anchorWithStickyNavbar_LWe7" id="a-cross-organizational-data-sharing-scenario">A cross organizational data sharing scenario<a href="#a-cross-organizational-data-sharing-scenario" class="hash-link" aria-label="Direct link to A cross organizational data sharing scenario" title="Direct link to A cross organizational data sharing scenario"></a></h2>
+<p>Our goal is the implementation of a cross organization decentralized data processing scenario, in which protected local data should be processed in combination with public data from public sources in a collaborative manner.
+Instead of copying all data into a central data lake or a central data platform we decided to use federated analytics.
+Apache Wayang is the tool we work with.
+In our case, the public data is hosted on publicly available websites or data pods.
+A client can use the HTTP(S) protocol to read the data which is given in a well defined format.
+For simplicity we decided to use CSV format.
+When we look into the data of each participant we have a different perspective.</p>
+<p>Our processing procedure should calculate a particular metric on the <em>local data</em> of each participant.
+An example of such a metric is the average spending of all users on a particular product category per month.
+This can vary from partner to partner, hence, we want to be able to calculate a peer-group comparison so that each partner can see its own metric compared with a global average calculated from contributions by all partners.
+Such a process requires global averaging and local averaging.
+And due to governance constraints, we can’t bring all raw data together in one place.</p>
+<p>Instead, we want to use Apache Wayang for this purpose.
+We simplify the procedure and split it into two phases.
+Phase one is the process, which allows each participant to calculate the local metrics.
+This requires only local data. The second phase requires data from all collaborating partners.
+The monthly sum and counter values per partner and category are needed in one place by all other parties.
+Hence, the algorithm of the first phase stores the local results locally, and the contributions to the global results in an externally accessible Kafka topic.
+We assume this is done by each of the partners.</p>
+<p>Now we have a scenario, in which an Apache Wayang process must be able to read data from multiple Apache Kafka topics from multiple Apache Kafka clusters but finally writes into a single Kafka topic, which then can be accessed by all the participating clients.</p>
+<p><img decoding="async" loading="lazy" alt="images/image-1.png" src="/assets/images/image-1-9cc35d5aea2b867d7e5759a96bd02334.png" width="904" height="550" class="img_ev3q"></p>
+<p>The illustration shows the data flows in such a scenario.
+Jobs with red border are executed by the participants in isolation within their own data processing environments.
+But they share some of the data, using publicly accessible Kafka topics, marked by A. Job 4 is the Apache Wayang job in our focus: here we intent to read data from 3 different source systems, and write results into a fourth system (marked as B), which can be accesses by all participants again.</p>
+<p>With this in mind we want to implement an Apache Wayang application which implements the illustrated <em>Job 4</em>.
+Since as of today, there is now <em>KafkaSource</em> and <em>KafkaSink</em> available in Apache Wayang, an implementation of both will be our first step.
+Our assumption is, that in the beginning, there won’t be much data.</p>
+<p>Apache Spark is not required to cope with the load, but we expect, that in the future, a single Java application would not be able to handle our workload.
+Hence, we want to utilize the Apache Wayang abstraction over multiple processing platforms, starting with Java.
+Later, we want to switch to Apache Spark.</p></div><footer class="row docusaurus-mt-lg"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/kafka">kafka</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/cross-organization-data-collaboration">cross organization [...]
<p> Apache Wayang is an effort undergoing incubation at The Apache Software Foundation (ASF), sponsored by the Apache Incubator. Incubation is required of all newly accepted projects until a further review indicates that the infrastructure, communications, and decision making process have stabilized in a manner consistent with other successful ASF projects. While incubation status is not necessarily a reflection of the completeness or stability of the code, it does indicate that th [...]
<p>
Copyright © 2024 The Apache Software Foundation, Licensed under the Apache License, Version 2.0. <br>
diff --git a/blog/kafka-meets-wayang-1/index.html b/blog/kafka-meets-wayang-1/index.html
index 8c78899b..62de3e52 100644
--- a/blog/kafka-meets-wayang-1/index.html
+++ b/blog/kafka-meets-wayang-1/index.html
@@ -3,20 +3,20 @@
<head>
<meta charset="UTF-8">
<meta name="generator" content="Docusaurus v3.1.1">
-<title data-rh="true">Apache Kafka meets Apache Wayang : Part 1 | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/blog/kafka-meets-wayang-1"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" nam [...]
+<title data-rh="true">Apache Kafka meets Wayang - Part 1 | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/blog/kafka-meets-wayang-1"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docu [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.01bb8940.js" defer="defer"></script>
-<script src="/assets/js/main.0cc2e93a.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.8489d39d.js" defer="defer"></script>
+<script src="/assets/js/main.4f5176d9.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
-<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
+<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
<p>This article is the first of a four part series about federated data analysis using Apache Wayang.
The first article starts with an introduction of a typical data colaboration scenario which will emerge in our digital future.</p>
<p>In part two and three we will share a summary of our Apache Kafka client implementation for Apache Wayang.
We started with the Java Platform (part 2) and the Apache Spark implementation follows (W.I.P.) in part three.</p>
-<p>The use case behind this work is an imaginary data collaboration scenario.</p>
-<p>We see this example and the demand for a solution already in many places.</p>
-<p>For us this is motivation enough to propose a solution.
+<p>The use case behind this work is an imaginary data collaboration scenario.
+We see this example and the demand for a solution already in many places.
+For us this is motivation enough to propose a solution.
This would also allow us to do more local data processing, and businesses can stop moving data around the world, but rather care about data locality while they expose and share specific information to others by using data federation.
This reduces complexity of data management and cost dramatically.</p>
<p>For this purpose, we illustrate a cross organizational data sharing scenario from the finance sector soon.
@@ -51,8 +51,8 @@ But they share some of the data, using publicly accessible Kafka topics, marked
Since as of today, there is now <em>KafkaSource</em> and <em>KafkaSink</em> available in Apache Wayang, an implementation of both will be our first step.
Our assumption is, that in the beginning, there won’t be much data.</p>
<p>Apache Spark is not required to cope with the load, but we expect, that in the future, a single Java application would not be able to handle our workload.
-Hence, we want to utilize the Apache Wayang abstraction over multiple processing platforms, starting with Java.</p>
-<p>Later, we want to switch to Apache Spark.</p></div><footer class="row docusaurus-mt-lg blogPostFooterDetailsFull_mRVl"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/kafka">kafka</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/cross-organization-data [...]
+Hence, we want to utilize the Apache Wayang abstraction over multiple processing platforms, starting with Java.
+Later, we want to switch to Apache Spark.</p></div><footer class="row docusaurus-mt-lg blogPostFooterDetailsFull_mRVl"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/kafka">kafka</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/cross-organization-data-co [...]
<p> Apache Wayang is an effort undergoing incubation at The Apache Software Foundation (ASF), sponsored by the Apache Incubator. Incubation is required of all newly accepted projects until a further review indicates that the infrastructure, communications, and decision making process have stabilized in a manner consistent with other successful ASF projects. While incubation status is not necessarily a reflection of the completeness or stability of the code, it does indicate that th [...]
<p>
Copyright © 2024 The Apache Software Foundation, Licensed under the Apache License, Version 2.0. <br>
diff --git a/blog/kafka-meets-wayang-2/index.html b/blog/kafka-meets-wayang-2/index.html
index 352a1d9e..4def5d4a 100644
--- a/blog/kafka-meets-wayang-2/index.html
+++ b/blog/kafka-meets-wayang-2/index.html
@@ -3,13 +3,13 @@
<head>
<meta charset="UTF-8">
<meta name="generator" content="Docusaurus v3.1.1">
-<title data-rh="true">Apache Kafka meets Apache Wayang - Part 2 | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/blog/kafka-meets-wayang-2"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" nam [...]
+<title data-rh="true">Apache Kafka meets Wayang - Part 2 | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/blog/kafka-meets-wayang-2"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docu [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.01bb8940.js" defer="defer"></script>
-<script src="/assets/js/main.0cc2e93a.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.8489d39d.js" defer="defer"></script>
+<script src="/assets/js/main.4f5176d9.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
-<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
+<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
We look into the “Read- and Write-Path” for our data items, called <em>DataQuanta</em>.</p>
<h2 class="anchor anchorWithStickyNavbar_LWe7" id="apache-wayangs-read--write-path-for-kafka-topics">Apache Wayang’s Read & Write Path for Kafka topics<a href="#apache-wayangs-read--write-path-for-kafka-topics" class="hash-link" aria-label="Direct link to Apache Wayang’s Read & Write Path for Kafka topics" title="Direct link to Apache Wayang’s Read & Write Path for Kafka topics"></a></h2>
<p>To describe the read and write paths for data in the context of the created Apache Wayang code snippet, the primary classes and interfaces we need to understand are as follows:</p>
@@ -33,8 +33,8 @@ We created the Kafka Source and Kafka Sink components so that our cross organiza
<p><strong>Level 1 – Wayang execution plan with abstract operators</strong></p>
<p>The implementation of our Kafka Source and Kafka Sink components for Apache Wayang requires new methods and classes on three layers.
First of all in the API package.
-Here we use the JavaPlanBuilder to expose the function for selecting a Kafka topic as the source to be used by client.<br>
-<!-- -->The class <em>JavaPlanBuilder</em> in package <em>org.apache.wayang.api</em> in the project <em>wayang-api/wayang-api-scala-java</em> exposes our new functionality to our external client.
+Here we use the JavaPlanBuilder to expose the function for selecting a Kafka topic as the source to be used by client.
+The class <em>JavaPlanBuilder</em> in package <em>org.apache.wayang.api</em> in the project <em>wayang-api/wayang-api-scala-java</em> exposes our new functionality to our external client.
An instance of the JavaPlanBuilder is used to define the data processing pipeline.
We use its <em>readKafkaTopic()</em> which specifies the source Kafka topic to read from, and for the write path we use the <em>writeKafkaTopic()</em> method.
Both Methods do only trigger activities in the background.</p>
@@ -49,8 +49,8 @@ In this API layer we use the Scala programming language, but we utilize the Java
<p>Therefore, the mappings package has a class <em>Mappings</em> in which all relevant input and output operators are listed.
We use it to register the KafkaSourceMapping and a KafkaSinkMapping for the particular platform, Java in our case.
These classes allow the Apache Wayang framework to use the Java implementation of the KafkaTopicSource component (and KafkaTopicSink respectively).
-While the Wayang execution plan uses the higher abstractions, here on the “platform level” we have to link the specific implementation for the target platform.</p>
-<p>In our case this leads to a Java program running on a JVM which is set up by the Apache Wayang framework using the logical components of the execution plan.</p>
+While the Wayang execution plan uses the higher abstractions, here on the “platform level” we have to link the specific implementation for the target platform.
+In our case this leads to a Java program running on a JVM which is set up by the Apache Wayang framework using the logical components of the execution plan.</p>
<p>Those mappings link the real implementation of our operators the ones used in an execution plan.
The JavaKafkaTopicSource and the JavaKafkaTopicSink extend the KafkaTopicSource and KafkaTopicSink so that the lower level implementation of those classes become available within Wayang’s Java Platform context.</p>
<p>In this layer, the KafkaConsumer class and the KafkaProducer class are used, but both are configured and instantiated in the next layer underneath.
@@ -65,8 +65,8 @@ Details about consumer and producers, client configuration, and schema handling
The layer above handles the mapping of the components at startup time.
All this wiring is needed to keep Wayang open and flexible so that multiple external systems can be used in a variety of combinations and using multiple target platforms in combinations.</p>
<h2 class="anchor anchorWithStickyNavbar_LWe7" id="outlook">Outlook<a href="#outlook" class="hash-link" aria-label="Direct link to Outlook" title="Direct link to Outlook"></a></h2>
-<p>The next part of the article series will cover the creation of an Kafka Source and Sink component for the Apache Spark platform, which allows our use case to scale.</p>
-<p>Finally, in part four we bring all puzzles together, and show the full implementation of the multi organizational data collaboration use case.</p></div><footer class="row docusaurus-mt-lg blogPostFooterDetailsFull_mRVl"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/kafka">kafka</a>< [...]
+<p>The next part of the article series will cover the creation of an Kafka Source and Sink component for the Apache Spark platform, which allows our use case to scale.
+Finally, in part four we bring all puzzles together, and show the full implementation of the multi organizational data collaboration use case.</p></div><footer class="row docusaurus-mt-lg blogPostFooterDetailsFull_mRVl"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/kafka">kafka</a></li [...]
<p> Apache Wayang is an effort undergoing incubation at The Apache Software Foundation (ASF), sponsored by the Apache Incubator. Incubation is required of all newly accepted projects until a further review indicates that the infrastructure, communications, and decision making process have stabilized in a manner consistent with other successful ASF projects. While incubation status is not necessarily a reflection of the completeness or stability of the code, it does indicate that th [...]
<p>
Copyright © 2024 The Apache Software Foundation, Licensed under the Apache License, Version 2.0. <br>
diff --git a/blog/kafka-meets-wayang-3/index.html b/blog/kafka-meets-wayang-3/index.html
index eb567ff1..5bc8a977 100644
--- a/blog/kafka-meets-wayang-3/index.html
+++ b/blog/kafka-meets-wayang-3/index.html
@@ -3,13 +3,13 @@
<head>
<meta charset="UTF-8">
<meta name="generator" content="Docusaurus v3.1.1">
-<title data-rh="true">Apache Kafka meets Apache Wayang - Part 3 | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/blog/kafka-meets-wayang-3"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" nam [...]
+<title data-rh="true">Apache Kafka meets Wayang - Part 3 | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/blog/kafka-meets-wayang-3"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docu [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.01bb8940.js" defer="defer"></script>
-<script src="/assets/js/main.0cc2e93a.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.8489d39d.js" defer="defer"></script>
+<script src="/assets/js/main.4f5176d9.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
-<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
+<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
Motivated by the learnings from last time, I stated implementing a Kafka Source component and a Kafka Sink component for the Apache Spark platform in Apache Wayang.
In our previous article we shared the results of the work on the frist Apache Kafka integration using the Java Platform.</p>
<p>Let's see how it goes this time with Apache Spark.</p>
diff --git a/blog/page/2/index.html b/blog/page/2/index.html
deleted file mode 100644
index 5e4c9a5e..00000000
--- a/blog/page/2/index.html
+++ /dev/null
@@ -1,63 +0,0 @@
-<!doctype html>
-<html lang="en" dir="ltr" class="blog-wrapper blog-list-page plugin-blog plugin-id-default" data-has-hydrated="false">
-<head>
-<meta charset="UTF-8">
-<meta name="generator" content="Docusaurus v3.1.1">
-<title data-rh="true">Blog | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/blog/page/2"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language" content="en"><meta data-rh=" [...]
-<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.01bb8940.js" defer="defer"></script>
-<script src="/assets/js/main.0cc2e93a.js" defer="defer"></script>
-</head>
-<body class="navigation-with-keyboard">
-<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
-<p>This article is the first of a four part series about federated data analysis using Apache Wayang.
-The first article starts with an introduction of a typical data colaboration scenario which will emerge in our digital future.</p>
-<p>In part two and three we will share a summary of our Apache Kafka client implementation for Apache Wayang.
-We started with the Java Platform (part 2) and the Apache Spark implementation follows (W.I.P.) in part three.</p>
-<p>The use case behind this work is an imaginary data collaboration scenario.
-We see this example and the demand for a solution already in many places.
-For us this is motivation enough to propose a solution.
-This would also allow us to do more local data processing, and businesses can stop moving data around the world, but rather care about data locality while they expose and share specific information to others by using data federation.
-This reduces complexity of data management and cost dramatically.</p>
-<p>For this purpose, we illustrate a cross organizational data sharing scenario from the finance sector soon.
-This analysis pattern will also be relevant in the context of data analysis along supply chains, another typical example where data from many stakeholder together is needed but never managed in one place, for good reasons.</p>
-<p>Data federation can help us to unlock the hidden value of all those isolated data lakes.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="a-cross-organizational-data-sharing-scenario">A cross organizational data sharing scenario<a href="#a-cross-organizational-data-sharing-scenario" class="hash-link" aria-label="Direct link to A cross organizational data sharing scenario" title="Direct link to A cross organizational data sharing scenario"></a></h2>
-<p>Our goal is the implementation of a cross organization decentralized data processing scenario, in which protected local data should be processed in combination with public data from public sources in a collaborative manner.
-Instead of copying all data into a central data lake or a central data platform we decided to use federated analytics.
-Apache Wayang is the tool we work with.
-In our case, the public data is hosted on publicly available websites or data pods.
-A client can use the HTTP(S) protocol to read the data which is given in a well defined format.
-For simplicity we decided to use CSV format.
-When we look into the data of each participant we have a different perspective.</p>
-<p>Our processing procedure should calculate a particular metric on the <em>local data</em> of each participant.
-An example of such a metric is the average spending of all users on a particular product category per month.
-This can vary from partner to partner, hence, we want to be able to calculate a peer-group comparison so that each partner can see its own metric compared with a global average calculated from contributions by all partners.
-Such a process requires global averaging and local averaging.
-And due to governance constraints, we can’t bring all raw data together in one place.</p>
-<p>Instead, we want to use Apache Wayang for this purpose.
-We simplify the procedure and split it into two phases.
-Phase one is the process, which allows each participant to calculate the local metrics.
-This requires only local data. The second phase requires data from all collaborating partners.
-The monthly sum and counter values per partner and category are needed in one place by all other parties.
-Hence, the algorithm of the first phase stores the local results locally, and the contributions to the global results in an externally accessible Kafka topic.
-We assume this is done by each of the partners.</p>
-<p>Now we have a scenario, in which an Apache Wayang process must be able to read data from multiple Apache Kafka topics from multiple Apache Kafka clusters but finally writes into a single Kafka topic, which then can be accessed by all the participating clients.</p>
-<p><img decoding="async" loading="lazy" alt="images/image-1.png" src="/assets/images/image-1-9cc35d5aea2b867d7e5759a96bd02334.png" width="904" height="550" class="img_ev3q"></p>
-<p>The illustration shows the data flows in such a scenario.
-Jobs with red border are executed by the participants in isolation within their own data processing environments.
-But they share some of the data, using publicly accessible Kafka topics, marked by A. Job 4 is the Apache Wayang job in our focus: here we intent to read data from 3 different source systems, and write results into a fourth system (marked as B), which can be accesses by all participants again.</p>
-<p>With this in mind we want to implement an Apache Wayang application which implements the illustrated <em>Job 4</em>.
-Since as of today, there is now <em>KafkaSource</em> and <em>KafkaSink</em> available in Apache Wayang, an implementation of both will be our first step.
-Our assumption is, that in the beginning, there won’t be much data.</p>
-<p>Apache Spark is not required to cope with the load, but we expect, that in the future, a single Java application would not be able to handle our workload.
-Hence, we want to utilize the Apache Wayang abstraction over multiple processing platforms, starting with Java.
-Later, we want to switch to Apache Spark.</p></div><footer class="row docusaurus-mt-lg"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/kafka">kafka</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/cross-organization-data-collaboration">cross organization [...]
- <p> Apache Wayang is an effort undergoing incubation at The Apache Software Foundation (ASF), sponsored by the Apache Incubator. Incubation is required of all newly accepted projects until a further review indicates that the infrastructure, communications, and decision making process have stabilized in a manner consistent with other successful ASF projects. While incubation status is not necessarily a reflection of the completeness or stability of the code, it does indicate that th [...]
- <p>
- Copyright © 2024 The Apache Software Foundation, Licensed under the Apache License, Version 2.0. <br>
- Apache, the names of Apache projects, and the feather logo are either registered trademarks or trademarks of the Apache Software Foundation in the United States and/or other countries.
- </p>
- </div></div></div></div></footer></div>
-</body>
-</html>
\ No newline at end of file
diff --git a/blog/rss.xml b/blog/rss.xml
index fd7c30af..2e1352ea 100644
--- a/blog/rss.xml
+++ b/blog/rss.xml
@@ -4,251 +4,137 @@
<title>Apache Wayang (incubating) Blog</title>
<link>https://wayang.apache.org/blog</link>
<description>Apache Wayang (incubating) Blog</description>
- <lastBuildDate>Sat, 21 Sep 2024 07:26:48 GMT</lastBuildDate>
+ <lastBuildDate>Sun, 01 Sep 2024 00:00:00 GMT</lastBuildDate>
<docs>https://validator.w3.org/feed/docs/rss2.html</docs>
<generator>https://github.com/jpmonette/feed</generator>
<language>en</language>
<item>
- <title><![CDATA[Apache Kafka meets Apache Wayang : Part 1]]></title>
- <link>https://wayang.apache.org/blog/kafka-meets-wayang-1</link>
- <guid>https://wayang.apache.org/blog/kafka-meets-wayang-1</guid>
- <pubDate>Sat, 21 Sep 2024 07:26:48 GMT</pubDate>
- <description><![CDATA[Intro]]></description>
- <content:encoded><![CDATA[<h2 class="anchor anchorWithStickyNavbar_LWe7" id="intro">Intro<a href="https://wayang.apache.org/blog/kafka-meets-wayang-1#intro" class="hash-link" aria-label="Direct link to Intro" title="Direct link to Intro"></a></h2>
-<p>This article is the first of a four part series about federated data analysis using Apache Wayang.
-The first article starts with an introduction of a typical data colaboration scenario which will emerge in our digital future.</p>
-<p>In part two and three we will share a summary of our Apache Kafka client implementation for Apache Wayang.
-We started with the Java Platform (part 2) and the Apache Spark implementation follows (W.I.P.) in part three.</p>
-<p>The use case behind this work is an imaginary data collaboration scenario.</p>
-<p>We see this example and the demand for a solution already in many places.</p>
-<p>For us this is motivation enough to propose a solution.
-This would also allow us to do more local data processing, and businesses can stop moving data around the world, but rather care about data locality while they expose and share specific information to others by using data federation.
-This reduces complexity of data management and cost dramatically.</p>
-<p>For this purpose, we illustrate a cross organizational data sharing scenario from the finance sector soon.
-This analysis pattern will also be relevant in the context of data analysis along supply chains, another typical example where data from many stakeholder together is needed but never managed in one place, for good reasons.</p>
-<p>Data federation can help us to unlock the hidden value of all those isolated data lakes.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="a-cross-organizational-data-sharing-scenario">A cross organizational data sharing scenario<a href="https://wayang.apache.org/blog/kafka-meets-wayang-1#a-cross-organizational-data-sharing-scenario" class="hash-link" aria-label="Direct link to A cross organizational data sharing scenario" title="Direct link to A cross organizational data sharing scenario"></a></h2>
-<p>Our goal is the implementation of a cross organization decentralized data processing scenario, in which protected local data should be processed in combination with public data from public sources in a collaborative manner.
-Instead of copying all data into a central data lake or a central data platform we decided to use federated analytics.
-Apache Wayang is the tool we work with.
-In our case, the public data is hosted on publicly available websites or data pods.
-A client can use the HTTP(S) protocol to read the data which is given in a well defined format.
-For simplicity we decided to use CSV format.
-When we look into the data of each participant we have a different perspective.</p>
-<p>Our processing procedure should calculate a particular metric on the <em>local data</em> of each participant.
-An example of such a metric is the average spending of all users on a particular product category per month.
-This can vary from partner to partner, hence, we want to be able to calculate a peer-group comparison so that each partner can see its own metric compared with a global average calculated from contributions by all partners.
-Such a process requires global averaging and local averaging.
-And due to governance constraints, we can’t bring all raw data together in one place.</p>
-<p>Instead, we want to use Apache Wayang for this purpose.
-We simplify the procedure and split it into two phases.
-Phase one is the process, which allows each participant to calculate the local metrics.
-This requires only local data. The second phase requires data from all collaborating partners.
-The monthly sum and counter values per partner and category are needed in one place by all other parties.
-Hence, the algorithm of the first phase stores the local results locally, and the contributions to the global results in an externally accessible Kafka topic.
-We assume this is done by each of the partners.</p>
-<p>Now we have a scenario, in which an Apache Wayang process must be able to read data from multiple Apache Kafka topics from multiple Apache Kafka clusters but finally writes into a single Kafka topic, which then can be accessed by all the participating clients.</p>
-<p><img decoding="async" loading="lazy" alt="images/image-1.png" src="https://wayang.apache.org/assets/images/image-1-9cc35d5aea2b867d7e5759a96bd02334.png" width="904" height="550" class="img_ev3q"></p>
-<p>The illustration shows the data flows in such a scenario.
-Jobs with red border are executed by the participants in isolation within their own data processing environments.
-But they share some of the data, using publicly accessible Kafka topics, marked by A. Job 4 is the Apache Wayang job in our focus: here we intent to read data from 3 different source systems, and write results into a fourth system (marked as B), which can be accesses by all participants again.</p>
-<p>With this in mind we want to implement an Apache Wayang application which implements the illustrated <em>Job 4</em>.
-Since as of today, there is now <em>KafkaSource</em> and <em>KafkaSink</em> available in Apache Wayang, an implementation of both will be our first step.
-Our assumption is, that in the beginning, there won’t be much data.</p>
-<p>Apache Spark is not required to cope with the load, but we expect, that in the future, a single Java application would not be able to handle our workload.
-Hence, we want to utilize the Apache Wayang abstraction over multiple processing platforms, starting with Java.</p>
-<p>Later, we want to switch to Apache Spark.</p>]]></content:encoded>
- <category>wayang</category>
- <category>kafka</category>
- <category>cross organization data collaboration</category>
- </item>
- <item>
- <title><![CDATA[Apache Kafka meets Apache Wayang - Part 2]]></title>
- <link>https://wayang.apache.org/blog/kafka-meets-wayang-2</link>
- <guid>https://wayang.apache.org/blog/kafka-meets-wayang-2</guid>
- <pubDate>Sat, 21 Sep 2024 07:26:48 GMT</pubDate>
- <description><![CDATA[In the second part of the article series we describe the implementation of the Kafka Source and Kafka Sink component for Apache Wayang.]]></description>
- <content:encoded><![CDATA[<p>In the second part of the article series we describe the implementation of the Kafka Source and Kafka Sink component for Apache Wayang.
-We look into the “Read- and Write-Path” for our data items, called <em>DataQuanta</em>.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="apache-wayangs-read--write-path-for-kafka-topics">Apache Wayang’s Read & Write Path for Kafka topics<a href="https://wayang.apache.org/blog/kafka-meets-wayang-2#apache-wayangs-read--write-path-for-kafka-topics" class="hash-link" aria-label="Direct link to Apache Wayang’s Read & Write Path for Kafka topics" title="Direct link to Apache Wayang’s Read & Write Path for Kafka topics"></a></h2>
-<p>To describe the read and write paths for data in the context of the created Apache Wayang code snippet, the primary classes and interfaces we need to understand are as follows:</p>
-<p><strong>WayangContext:</strong> This class is essential for initializing the Wayang processing environment.
-It allows you to configure the execution environment and register plugins that define which platforms Wayang can use for data processing tasks, such as <em>Java.basicPlugin()</em> for local Java execution.</p>
-<p><strong>JavaPlanBuilder:</strong> This class is used to build and define the data processing pipeline (or plan) in Wayang.
-It provides a fluent API to specify the operations to be performed on the data, from reading the input to processing it and writing the output.</p>
-<h3 class="anchor anchorWithStickyNavbar_LWe7" id="read-path">Read Path<a href="https://wayang.apache.org/blog/kafka-meets-wayang-2#read-path" class="hash-link" aria-label="Direct link to Read Path" title="Direct link to Read Path"></a></h3>
-<p>The read path describes how data is ingested from a source into the Wayang processing pipeline:</p>
-<p><em>Reading from Kafka Topic:</em> The method <em>readKafkaTopic(topicName)</em> is used to ingest data from a specified Kafka topic.
-This is the starting point of the data processing pipeline, where topicName represents the name of the Kafka topic from which data is read.</p>
-<p><em>Data Tokenization and Preparation:</em> Once the data is read from Kafka, it undergoes several transformations such as Splitting, Filtering, and Mapping.
-What follows are the procedures known as Reducing, Grouping, Co-Grouping, and Counting.</p>
-<h3 class="anchor anchorWithStickyNavbar_LWe7" id="write-path">Write Path<a href="https://wayang.apache.org/blog/kafka-meets-wayang-2#write-path" class="hash-link" aria-label="Direct link to Write Path" title="Direct link to Write Path"></a></h3>
-<p><em>Writing to Kafka Topic:</em> The final step in the pipeline involves writing the processed data back to a Kafka topic using <em>.writeKafkaTopic(...)</em>.
-This method takes parameters that specify the target Kafka topic, a serialization function to format the data as strings, and additional configuration for load profile estimation, which optimizes the writing process.</p>
-<p>This read-write path provides a comprehensive flow of data from ingestion from Kafka, through various processing steps, and finally back to Kafka, showcasing a full cycle of data processing within Apache Wayang's abstracted environment and is implemented in our example program shown in <em>listing 1</em>.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="implementation-of-input--and-output-operators">Implementation of Input- and Output Operators<a href="https://wayang.apache.org/blog/kafka-meets-wayang-2#implementation-of-input--and-output-operators" class="hash-link" aria-label="Direct link to Implementation of Input- and Output Operators" title="Direct link to Implementation of Input- and Output Operators"></a></h2>
-<p>The next section shows how a new pair of operators can be implemented to extend Apache Wayang’s capabilities on the input and output side.
-We created the Kafka Source and Kafka Sink components so that our cross organizational data collaboration scenario can be implemented using data streaming infrastructure.</p>
-<p><strong>Level 1 – Wayang execution plan with abstract operators</strong></p>
-<p>The implementation of our Kafka Source and Kafka Sink components for Apache Wayang requires new methods and classes on three layers.
-First of all in the API package.
-Here we use the JavaPlanBuilder to expose the function for selecting a Kafka topic as the source to be used by client.<br>
-<!-- -->The class <em>JavaPlanBuilder</em> in package <em>org.apache.wayang.api</em> in the project <em>wayang-api/wayang-api-scala-java</em> exposes our new functionality to our external client.
-An instance of the JavaPlanBuilder is used to define the data processing pipeline.
-We use its <em>readKafkaTopic()</em> which specifies the source Kafka topic to read from, and for the write path we use the <em>writeKafkaTopic()</em> method.
-Both Methods do only trigger activities in the background.</p>
-<p>For the output side, we use the <em>DataQuantaBuilder</em> class, which offers an implementation of the writeKafkaTopic function.
-This function is designed to send processed data, referred to as DataQuanta, to a specified Kafka topic.
-Essentially, it marks the final step in a data processing sequence constructed using the Apache Wayang framework.</p>
-<p>In the DataQuanta class we implemented the methods writeKafkaTopic and writeKafkaTopicJava which use the KafkaTopicSink class.
-In this API layer we use the Scala programming language, but we utilize the Java classes, implemented in the layer below.</p>
-<p><strong>Level 2 – Wiring between Platform Abstraction and Implementation</strong></p>
-<p>The second layer builds the bridge between the WayangContext and PlanBuilders which work together with DataQuanta and the DataQuantaBuilder.</p>
-<p>Also, the mapping between the abstract components and the specific implementations are defined in this layer.</p>
-<p>Therefore, the mappings package has a class <em>Mappings</em> in which all relevant input and output operators are listed.
-We use it to register the KafkaSourceMapping and a KafkaSinkMapping for the particular platform, Java in our case.
-These classes allow the Apache Wayang framework to use the Java implementation of the KafkaTopicSource component (and KafkaTopicSink respectively).
-While the Wayang execution plan uses the higher abstractions, here on the “platform level” we have to link the specific implementation for the target platform.</p>
-<p>In our case this leads to a Java program running on a JVM which is set up by the Apache Wayang framework using the logical components of the execution plan.</p>
-<p>Those mappings link the real implementation of our operators the ones used in an execution plan.
-The JavaKafkaTopicSource and the JavaKafkaTopicSink extend the KafkaTopicSource and KafkaTopicSink so that the lower level implementation of those classes become available within Wayang’s Java Platform context.</p>
-<p>In this layer, the KafkaConsumer class and the KafkaProducer class are used, but both are configured and instantiated in the next layer underneath.
-All this is done in the project <em>wayang-plarforms/wayang-java</em>.</p>
-<p><strong>Layer 3 – Input/Output Connector Layer</strong></p>
-<p>The <em>KafkaTopicSource</em> and <em>KafkaTopicSink</em> classes build the third layer of our implementation.
-Both are implemented in Java programming language.
-In this layer, the real Kafka-Client logic is defined.
-Details about consumer and producers, client configuration, and schema handling have to be handled here.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="summary">Summary<a href="https://wayang.apache.org/blog/kafka-meets-wayang-2#summary" class="hash-link" aria-label="Direct link to Summary" title="Direct link to Summary"></a></h2>
-<p>Both classes in the third layer implement the Kafka client logic which is needed by the Wayang-execution plan when external data flows should be established.
-The layer above handles the mapping of the components at startup time.
-All this wiring is needed to keep Wayang open and flexible so that multiple external systems can be used in a variety of combinations and using multiple target platforms in combinations.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="outlook">Outlook<a href="https://wayang.apache.org/blog/kafka-meets-wayang-2#outlook" class="hash-link" aria-label="Direct link to Outlook" title="Direct link to Outlook"></a></h2>
-<p>The next part of the article series will cover the creation of an Kafka Source and Sink component for the Apache Spark platform, which allows our use case to scale.</p>
-<p>Finally, in part four we bring all puzzles together, and show the full implementation of the multi organizational data collaboration use case.</p>]]></content:encoded>
- <category>wayang</category>
- <category>kafka</category>
- <category>cross organization data collaboration</category>
- </item>
- <item>
- <title><![CDATA[Apache Kafka meets Apache Wayang - Part 3]]></title>
- <link>https://wayang.apache.org/blog/kafka-meets-wayang-2</link>
- <guid>https://wayang.apache.org/blog/kafka-meets-wayang-2</guid>
- <pubDate>Sat, 21 Sep 2024 07:26:48 GMT</pubDate>
- <description><![CDATA[* THIS ARTICLE IS WORK IN PROGRESS *]]></description>
- <content:encoded><![CDATA[<p>In the second part of the article series we describe the implementation of the Kafka Source and Kafka Sink component for Apache Wayang.
-We look into the “Read- and Write-Path” for our data items, called <em>DataQuanta</em>.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="apache-wayangs-read--write-path-for-kafka-topics">Apache Wayang’s Read & Write Path for Kafka topics<a href="https://wayang.apache.org/blog/kafka-meets-wayang-2#apache-wayangs-read--write-path-for-kafka-topics" class="hash-link" aria-label="Direct link to Apache Wayang’s Read & Write Path for Kafka topics" title="Direct link to Apache Wayang’s Read & Write Path for Kafka topics"></a></h2>
-<p>To describe the read and write paths for data in the context of the created Apache Wayang code snippet, the primary classes and interfaces we need to understand are as follows:</p>
-<p><strong>WayangContext:</strong> This class is essential for initializing the Wayang processing environment.
-It allows you to configure the execution environment and register plugins that define which platforms Wayang can use for data processing tasks, such as <em>Java.basicPlugin()</em> for local Java execution.</p>
-<p><strong>JavaPlanBuilder:</strong> This class is used to build and define the data processing pipeline (or plan) in Wayang.
-It provides a fluent API to specify the operations to be performed on the data, from reading the input to processing it and writing the output.</p>
-<h3 class="anchor anchorWithStickyNavbar_LWe7" id="read-path">Read Path<a href="https://wayang.apache.org/blog/kafka-meets-wayang-2#read-path" class="hash-link" aria-label="Direct link to Read Path" title="Direct link to Read Path"></a></h3>
-<p>The read path describes how data is ingested from a source into the Wayang processing pipeline:</p>
-<p><em>Reading from Kafka Topic:</em> The method <em>readKafkaTopic(topicName)</em> is used to ingest data from a specified Kafka topic.
-This is the starting point of the data processing pipeline, where topicName represents the name of the Kafka topic from which data is read.</p>
-<p><em>Data Tokenization and Preparation:</em> Once the data is read from Kafka, it undergoes several transformations such as Splitting, Filtering, and Mapping.
-What follows are the procedures known as Reducing, Grouping, Co-Grouping, and Counting.</p>
-<h3 class="anchor anchorWithStickyNavbar_LWe7" id="write-path">Write Path<a href="https://wayang.apache.org/blog/kafka-meets-wayang-2#write-path" class="hash-link" aria-label="Direct link to Write Path" title="Direct link to Write Path"></a></h3>
-<p><em>Writing to Kafka Topic:</em> The final step in the pipeline involves writing the processed data back to a Kafka topic using <em>.writeKafkaTopic(...)</em>.
-This method takes parameters that specify the target Kafka topic, a serialization function to format the data as strings, and additional configuration for load profile estimation, which optimizes the writing process.</p>
-<p>This read-write path provides a comprehensive flow of data from ingestion from Kafka, through various processing steps, and finally back to Kafka, showcasing a full cycle of data processing within Apache Wayang's abstracted environment and is implemented in our example program shown in <em>listing 1</em>.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="implementation-of-input--and-output-operators">Implementation of Input- and Output Operators<a href="https://wayang.apache.org/blog/kafka-meets-wayang-2#implementation-of-input--and-output-operators" class="hash-link" aria-label="Direct link to Implementation of Input- and Output Operators" title="Direct link to Implementation of Input- and Output Operators"></a></h2>
-<p>The next section shows how a new pair of operators can be implemented to extend Apache Wayang’s capabilities on the input and output side.
-We created the Kafka Source and Kafka Sink components so that our cross organizational data collaboration scenario can be implemented using data streaming infrastructure.</p>
-<p><strong>Level 1 – Wayang execution plan with abstract operators</strong></p>
-<p>The implementation of our Kafka Source and Kafka Sink components for Apache Wayang requires new methods and classes on three layers.
-First of all in the API package.
-Here we use the JavaPlanBuilder to expose the function for selecting a Kafka topic as the source to be used by client.<br>
-<!-- -->The class <em>JavaPlanBuilder</em> in package <em>org.apache.wayang.api</em> in the project <em>wayang-api/wayang-api-scala-java</em> exposes our new functionality to our external client.
-An instance of the JavaPlanBuilder is used to define the data processing pipeline.
-We use its <em>readKafkaTopic()</em> which specifies the source Kafka topic to read from, and for the write path we use the <em>writeKafkaTopic()</em> method.
-Both Methods do only trigger activities in the background.</p>
-<p>For the output side, we use the <em>DataQuantaBuilder</em> class, which offers an implementation of the writeKafkaTopic function.
-This function is designed to send processed data, referred to as DataQuanta, to a specified Kafka topic.
-Essentially, it marks the final step in a data processing sequence constructed using the Apache Wayang framework.</p>
-<p>In the DataQuanta class we implemented the methods writeKafkaTopic and writeKafkaTopicJava which use the KafkaTopicSink class.
-In this API layer we use the Scala programming language, but we utilize the Java classes, implemented in the layer below.</p>
-<p><strong>Level 2 – Wiring between Platform Abstraction and Implementation</strong></p>
-<p>The second layer builds the bridge between the WayangContext and PlanBuilders which work together with DataQuanta and the DataQuantaBuilder.</p>
-<p>Also, the mapping between the abstract components and the specific implementations are defined in this layer.</p>
-<p>Therefore, the mappings package has a class <em>Mappings</em> in which all relevant input and output operators are listed.
-We use it to register the KafkaSourceMapping and a KafkaSinkMapping for the particular platform, Java in our case.
-These classes allow the Apache Wayang framework to use the Java implementation of the KafkaTopicSource component (and KafkaTopicSink respectively).
-While the Wayang execution plan uses the higher abstractions, here on the “platform level” we have to link the specific implementation for the target platform.</p>
-<p>In our case this leads to a Java program running on a JVM which is set up by the Apache Wayang framework using the logical components of the execution plan.</p>
-<p>Those mappings link the real implementation of our operators the ones used in an execution plan.
-The JavaKafkaTopicSource and the JavaKafkaTopicSink extend the KafkaTopicSource and KafkaTopicSink so that the lower level implementation of those classes become available within Wayang’s Java Platform context.</p>
-<p>In this layer, the KafkaConsumer class and the KafkaProducer class are used, but both are configured and instantiated in the next layer underneath.
-All this is done in the project <em>wayang-plarforms/wayang-java</em>.</p>
-<p><strong>Layer 3 – Input/Output Connector Layer</strong></p>
-<p>The <em>KafkaTopicSource</em> and <em>KafkaTopicSink</em> classes build the third layer of our implementation.
-Both are implemented in Java programming language.
-In this layer, the real Kafka-Client logic is defined.
-Details about consumer and producers, client configuration, and schema handling have to be handled here.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="summary">Summary<a href="https://wayang.apache.org/blog/kafka-meets-wayang-2#summary" class="hash-link" aria-label="Direct link to Summary" title="Direct link to Summary"></a></h2>
-<p>Both classes in the third layer implement the Kafka client logic which is needed by the Wayang-execution plan when external data flows should be established.
-The layer above handles the mapping of the components at startup time.
-All this wiring is needed to keep Wayang open and flexible so that multiple external systems can be used in a variety of combinations and using multiple target platforms in combinations.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="outlook">Outlook<a href="https://wayang.apache.org/blog/kafka-meets-wayang-2#outlook" class="hash-link" aria-label="Direct link to Outlook" title="Direct link to Outlook"></a></h2>
-<p>The next part of the article series will cover the creation of an Kafka Source and Sink component for the Apache Spark platform, which allows our use case to scale.</p>
-<p>Finally, in part four we bring all puzzles together, and show the full implementation of the multi organizational data collaboration use case.</p>]]></content:encoded>
- <category>wayang</category>
- <category>kafka</category>
- <category>spark</category>
- <category>cross organization data collaboration</category>
- </item>
- <item>
- <title><![CDATA[Apache Kafka meets Apache Wayang - Part 1]]></title>
- <link>https://wayang.apache.org/blog/kafka-meets-wayang-1</link>
- <guid>https://wayang.apache.org/blog/kafka-meets-wayang-1</guid>
- <pubDate>Sat, 21 Sep 2024 07:26:48 GMT</pubDate>
+ <title><![CDATA[Apache Wayang Release Odysse]]></title>
+ <link>https://wayang.apache.org/blog/wayang-release-odysse</link>
+ <guid>https://wayang.apache.org/blog/wayang-release-odysse</guid>
+ <pubDate>Sun, 01 Sep 2024 00:00:00 GMT</pubDate>
<description><![CDATA[Intro]]></description>
- <content:encoded><![CDATA[<h2 class="anchor anchorWithStickyNavbar_LWe7" id="intro">Intro<a href="https://wayang.apache.org/blog/kafka-meets-wayang-1#intro" class="hash-link" aria-label="Direct link to Intro" title="Direct link to Intro"></a></h2>
-<p>This article is the first of a four part series about federated data analysis using Apache Wayang.
-The first article starts with an introduction of a typical data colaboration scenario which will emerge in our digital future.</p>
-<p>In part two and three we will share a summary of our Apache Kafka client implementation for Apache Wayang.
-We started with the Java Platform (part 2) and the Apache Spark implementation follows (W.I.P.) in part three.</p>
-<p>The use case behind this work is an imaginary data collaboration scenario.</p>
-<p>We see this example and the demand for a solution already in many places.</p>
-<p>For us this is motivation enough to propose a solution.
-This would also allow us to do more local data processing, and businesses can stop moving data around the world, but rather care about data locality while they expose and share specific information to others by using data federation.
-This reduces complexity of data management and cost dramatically.</p>
-<p>For this purpose, we illustrate a cross organizational data sharing scenario from the finance sector soon.
-This analysis pattern will also be relevant in the context of data analysis along supply chains, another typical example where data from many stakeholder together is needed but never managed in one place, for good reasons.</p>
-<p>Data federation can help us to unlock the hidden value of all those isolated data lakes.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="a-cross-organizational-data-sharing-scenario">A cross organizational data sharing scenario<a href="https://wayang.apache.org/blog/kafka-meets-wayang-1#a-cross-organizational-data-sharing-scenario" class="hash-link" aria-label="Direct link to A cross organizational data sharing scenario" title="Direct link to A cross organizational data sharing scenario"></a></h2>
-<p>Our goal is the implementation of a cross organization decentralized data processing scenario, in which protected local data should be processed in combination with public data from public sources in a collaborative manner.
-Instead of copying all data into a central data lake or a central data platform we decided to use federated analytics.
-Apache Wayang is the tool we work with.
-In our case, the public data is hosted on publicly available websites or data pods.
-A client can use the HTTP(S) protocol to read the data which is given in a well defined format.
-For simplicity we decided to use CSV format.
-When we look into the data of each participant we have a different perspective.</p>
-<p>Our processing procedure should calculate a particular metric on the <em>local data</em> of each participant.
-An example of such a metric is the average spending of all users on a particular product category per month.
-This can vary from partner to partner, hence, we want to be able to calculate a peer-group comparison so that each partner can see its own metric compared with a global average calculated from contributions by all partners.
-Such a process requires global averaging and local averaging.
-And due to governance constraints, we can’t bring all raw data together in one place.</p>
-<p>Instead, we want to use Apache Wayang for this purpose.
-We simplify the procedure and split it into two phases.
-Phase one is the process, which allows each participant to calculate the local metrics.
-This requires only local data. The second phase requires data from all collaborating partners.
-The monthly sum and counter values per partner and category are needed in one place by all other parties.
-Hence, the algorithm of the first phase stores the local results locally, and the contributions to the global results in an externally accessible Kafka topic.
-We assume this is done by each of the partners.</p>
-<p>Now we have a scenario, in which an Apache Wayang process must be able to read data from multiple Apache Kafka topics from multiple Apache Kafka clusters but finally writes into a single Kafka topic, which then can be accessed by all the participating clients.</p>
-<p><img decoding="async" loading="lazy" alt="images/image-1.png" src="https://wayang.apache.org/assets/images/image-1-9cc35d5aea2b867d7e5759a96bd02334.png" width="904" height="550" class="img_ev3q"></p>
-<p>The illustration shows the data flows in such a scenario.
-Jobs with red border are executed by the participants in isolation within their own data processing environments.
-But they share some of the data, using publicly accessible Kafka topics, marked by A. Job 4 is the Apache Wayang job in our focus: here we intent to read data from 3 different source systems, and write results into a fourth system (marked as B), which can be accesses by all participants again.</p>
-<p>With this in mind we want to implement an Apache Wayang application which implements the illustrated <em>Job 4</em>.
-Since as of today, there is now <em>KafkaSource</em> and <em>KafkaSink</em> available in Apache Wayang, an implementation of both will be our first step.
-Our assumption is, that in the beginning, there won’t be much data.</p>
-<p>Apache Spark is not required to cope with the load, but we expect, that in the future, a single Java application would not be able to handle our workload.
-Hence, we want to utilize the Apache Wayang abstraction over multiple processing platforms, starting with Java.</p>
-<p>Later, we want to switch to Apache Spark.</p>]]></content:encoded>
+ <content:encoded><![CDATA[<h2 class="anchor anchorWithStickyNavbar_LWe7" id="intro">Intro<a href="https://wayang.apache.org/blog/wayang-release-odysse#intro" class="hash-link" aria-label="Direct link to Intro" title="Direct link to Intro"></a></h2>
+<p>The ASF provides a robust infrastructure for open communities of software developers. We can share ideas, combine forces, contribute code, docs, review-energy, art work, and from time to time we can nail it down. A release defines an intermediate result of the continuous community work.</p>
+<p>How we do such a release in the Apache Wayang team is an essential aspect towards graduation. First of all, there are some references to take into account, such as:</p>
+<ul>
+<li><a href="https://www.apache.org/legal/release-policy.html" target="_blank" rel="noopener noreferrer">https://www.apache.org/legal/release-policy.html</a></li>
+<li><a href="https://maven.apache.org/maven-release/maven-release-plugin/" target="_blank" rel="noopener noreferrer">https://maven.apache.org/maven-release/maven-release-plugin/</a></li>
+</ul>
+<p>Assuming you are (P)PMC, and assuming that you have the right permissions for such a release, you can follow the path as described in this guide:</p>
+<ul>
+<li><a href="https://plc4x.apache.org/developers/release/release.html" target="_blank" rel="noopener noreferrer">https://plc4x.apache.org/developers/release/release.html</a></li>
+</ul>
+<p>I tried to follow exactly this procedure, several times. I failed. Here I share the current status of my <strong>release attempts</strong>.</p>
+<p>I plan a longer tour, and do not want to block the project for a long time.
+Hence I create this draft, and I hope we can unblock this project as soon as possible.</p>
+<h2 class="anchor anchorWithStickyNavbar_LWe7" id="status">Status:<a href="https://wayang.apache.org/blog/wayang-release-odysse#status" class="hash-link" aria-label="Direct link to Status:" title="Direct link to Status:"></a></h2>
+<p>I am not able to conduct the <em>mvn release:perform</em> step.
+Anything before worked, sometimes only after some digging, but it worked.</p>
+<ul>
+<li>
+<p>We assume, that due to my membership in two ASF incubator projects I am not able to upload the artefacts to the Nexus repository (H1).</p>
+</li>
+<li>
+<p>It can be, that I have not the correct user and password in my <em>settings.xml</em> file (H2).</p>
+</li>
+</ul>
+<p>But I tested a manual login to the nexus server <a href="https://repository.apache.org/service/local/staging/deploy/maven2" target="_blank" rel="noopener noreferrer">https://repository.apache.org/service/local/staging/deploy/maven2</a> with success. And beyond that I have no idea how I can verify this detail alone.</p>
+<h2 class="anchor anchorWithStickyNavbar_LWe7" id="idea--proposal">Idea / Proposal<a href="https://wayang.apache.org/blog/wayang-release-odysse#idea--proposal" class="hash-link" aria-label="Direct link to Idea / Proposal" title="Direct link to Idea / Proposal"></a></h2>
+<p>(1) It would be great, if someone - who has done a release in any other ASF project or in Apache Wayang - could follow the steps I share, so that we can check where the problem hides itself.</p>
+<p>(2) As a follow-up task, I suggest to add a <strong>Release Guide</strong> to the Apache Wayang project, including release manager onboarding steps, and checklists for the particular project, derived from the referenced sources which are listed above.</p>
+<p>But for now it is all about sharing the status (as I did serveral times on multiple chanels, including JIRA, Slack, Mailing lists) and finding a solution for Apache Wayang release 1.0.</p>
+<h2 class="anchor anchorWithStickyNavbar_LWe7" id="latest-error">Latest Error:<a href="https://wayang.apache.org/blog/wayang-release-odysse#latest-error" class="hash-link" aria-label="Direct link to Latest Error:" title="Direct link to Latest Error:"></a></h2>
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn release:perform -X -DskipTests</span><br></span></code></pre><div class="buttonGroup__atx"><button ty [...]
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">[INFO] Caused by: org.eclipse.aether.deployment.DeploymentException: Failed to deploy artifacts: Could no [...]
+<h2 class="anchor anchorWithStickyNavbar_LWe7" id="activity-log">Activity Log<a href="https://wayang.apache.org/blog/wayang-release-odysse#activity-log" class="hash-link" aria-label="Direct link to Activity Log" title="Direct link to Activity Log"></a></h2>
+<div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn release:clean</span><br></span><span class="token-line" style="color:#393A34"><span cla [...]
+<div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn release:prepare </span><span class="token parameter variable" style="color:#36acaa">-Da [...]
+<div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token punctuation" style="color:#393A34">[</span><span class="token plain">ERROR</span><span class="toke [...]
+<div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn versions:set </span><span class="token parameter variable" style="color:#36acaa">-DnewV [...]
+<div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn release:prepare </span><span class="token parameter variable" style="color:#36acaa">-Da [...]
+<div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token punctuation" style="color:#393A34">[</span><span class="token plain">ERROR</span><span class="toke [...]
+<div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token function" style="color:#d73a49">git</span><span class="token plain"> status</span><br></span><span [...]
+<div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn release:prepare </span><span class="token parameter variable" style="color:#36acaa">-Da [...]
+<div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">Caused by: org.eclipse.aether.transfer.NoRepositoryConnectorException: Blocked mirror </spa [...]
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="dependency-on-jdk-11-during-release">Dependency on JDK-11 during release<a href="https://wayang.apache.org/blog/wayang-release-odysse#dependency-on-jdk-11-during-release" class="hash-link" aria-label="Direct link to Dependency on JDK-11 during release" title="Direct link to Dependency on JDK-11 during release"></a></h3>
+<blockquote>
+<p>FIXED with local JDK11 Setup</p>
+</blockquote>
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">brew install openjdk@11</span><br></span><span class="token-line" style="color:#393A34"><span class="toke [...]
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="manual-update-of-release-version">Manual update of release-version<a href="https://wayang.apache.org/blog/wayang-release-odysse#manual-update-of-release-version" class="hash-link" aria-label="Direct link to Manual update of release-version" title="Direct link to Manual update of release-version"></a></h3>
+<p>During the release procedure, do I have to set the version here in this configuration section manually?</p>
+<div class="language-xml codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-xml codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token tag punctuation" style="color:#393A34"><</span><span class="token tag" style="color:#00009f">plug [...]
+<blockquote>
+<p>It seems that these properties must be updated manually.</p>
+</blockquote>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="warning-regarding-illegal-reflective-access-operation">Warning regarding "illegal reflective access operation"<a href="https://wayang.apache.org/blog/wayang-release-odysse#warning-regarding-illegal-reflective-access-operation" class="hash-link" aria-label="Direct link to Warning regarding "illegal reflective access operation"" title="Direct link to Warning regarding "illegal reflective access operation""></a></h3>
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">[ERROR] WARNING: An illegal reflective access operation has occurred</span><br></span><span class="token- [...]
+<blockquote>
+<p><strong>This is still an OPEN ISSUE !</strong></p>
+</blockquote>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="rat-check-fails">RAT Check fails<a href="https://wayang.apache.org/blog/wayang-release-odysse#rat-check-fails" class="hash-link" aria-label="Direct link to RAT Check fails" title="Direct link to RAT Check fails"></a></h3>
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">[INFO] [ERROR] Failed to execute goal org.apache.rat:apache-rat-plugin:0.13:check (license-check) on proj [...]
+<div class="language-cat codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-cat codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#393A34" [...]
+<blockquote>
+<p><em>FIXED by adding .java-versions to .gitignore</em></p>
+</blockquote>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="tag-could-not-be-created-in-scm">Tag could not be created in SCM.<a href="https://wayang.apache.org/blog/wayang-release-odysse#tag-could-not-be-created-in-scm" class="hash-link" aria-label="Direct link to Tag could not be created in SCM." title="Direct link to Tag could not be created in SCM."></a></h3>
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">[ERROR] Failed to execute goal org.apache.maven.plugins:maven-release-plugin:3.0.1:prepare (default-cli) [...]
+<blockquote>
+<p>FIXED by manual changes in pom.xml.</p>
+</blockquote>
+<div class="language-xml codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-xml codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#393A34" [...]
+<div class="language-xml codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-xml codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token tag punctuation" style="color:#393A34"><</span><span class="token tag" style="color:#00009f">conf [...]
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn release:prepare -Darguments='-DskipTests=True -Dresume=True'</span><br></span></code></pre><div class [...]
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn clean package</span><br></span><span class="token-line" style="color:#393A34"><span class="token plai [...]
+<blockquote>
+<p>So far so good. But now the sun went down.</p>
+</blockquote>
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">[INFO] [ERROR] Failed to execute goal org.apache.maven.plugins:maven-deploy-plugin:3.0.0-M1:deploy (defau [...]
+<p>The null pointer exception indicates problems in <em>settings.xml</em>. This could be fixed.
+But now, the error changes with and I am still not able to stage the build results.</p>
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn release:perform -X -DskipTests</span><br></span></code></pre><div class="buttonGroup__atx"><button ty [...]
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">[INFO] Caused by: org.eclipse.aether.deployment.DeploymentException: Failed to deploy artifacts: Could no [...]
+<h2 class="anchor anchorWithStickyNavbar_LWe7" id="checklist-for-next-iteration">Checklist for next iteration:<a href="https://wayang.apache.org/blog/wayang-release-odysse#checklist-for-next-iteration" class="hash-link" aria-label="Direct link to Checklist for next iteration:" title="Direct link to Checklist for next iteration:"></a></h2>
+<p>The error you're encountering indicates that there is a problem with deploying the artifacts using the <code>maven-deploy-plugin</code>. The root cause of the error is a <code>NullPointerException</code> during the deployment process, specifically related to the <code>org.eclipse.aether.transfer.ArtifactTransferException</code>.</p>
+<p>Here's a step-by-step guide to troubleshoot and resolve this issue:</p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="1-check-maven-settings">1. Check Maven Settings<a href="https://wayang.apache.org/blog/wayang-release-odysse#1-check-maven-settings" class="hash-link" aria-label="Direct link to 1. Check Maven Settings" title="Direct link to 1. Check Maven Settings"></a></h3>
+<p>Ensure that your Maven settings (<code>settings.xml</code>) are correctly configured for deployment. Verify that the repository settings and credentials are correctly specified.</p>
+<p><strong>[DONE]</strong></p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="2-verify-repository-url">2. Verify Repository URL<a href="https://wayang.apache.org/blog/wayang-release-odysse#2-verify-repository-url" class="hash-link" aria-label="Direct link to 2. Verify Repository URL" title="Direct link to 2. Verify Repository URL"></a></h3>
+<p>Make sure the repository URL in your <code>pom.xml</code> or <code>settings.xml</code> is correct and reachable. The URL should point to the correct staging repository for deployment.</p>
+<p><em>[OPEN]</em> - I did not touch it, so I guess it is correct in the <em>pom.xml</em>.</p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="3-maven-version-compatibility">3. Maven Version Compatibility<a href="https://wayang.apache.org/blog/wayang-release-odysse#3-maven-version-compatibility" class="hash-link" aria-label="Direct link to 3. Maven Version Compatibility" title="Direct link to 3. Maven Version Compatibility"></a></h3>
+<p>Verify that you are using a compatible version of Maven. Sometimes, upgrading or downgrading Maven can resolve such issues.</p>
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">Apache Maven 3.9.6 (bc0240f3c744dd6b6ec2920b3cd08dcc295161ae)</span><br></span><span class="token-line" s [...]
+<p><strong>[DONE]</strong></p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="4-check-for-network-issues">4. Check for Network Issues<a href="https://wayang.apache.org/blog/wayang-release-odysse#4-check-for-network-issues" class="hash-link" aria-label="Direct link to 4. Check for Network Issues" title="Direct link to 4. Check for Network Issues"></a></h3>
+<p>Ensure there are no network issues that might be causing problems in connecting to the repository. Sometimes, network configurations, firewalls, or proxy settings can interfere with the deployment process.</p>
+<p><strong>[DONE]</strong></p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="5-update-maven-plugins">5. Update Maven Plugins<a href="https://wayang.apache.org/blog/wayang-release-odysse#5-update-maven-plugins" class="hash-link" aria-label="Direct link to 5. Update Maven Plugins" title="Direct link to 5. Update Maven Plugins"></a></h3>
+<p>Ensure that you are using the latest versions of the Maven plugins. Sometimes, bugs in older versions can cause unexpected issues.</p>
+<p><em>[OPEN]</em></p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="6-configure-the-maven-deploy-plugin-in-pomxml">6. Configure the <code>maven-deploy-plugin</code> in <code>pom.xml</code><a href="https://wayang.apache.org/blog/wayang-release-odysse#6-configure-the-maven-deploy-plugin-in-pomxml" class="hash-link" aria-label="Direct link to 6-configure-the-maven-deploy-plugin-in-pomxml" title="Direct link to 6-configure-the-maven-deploy-plugin-in-pomxml"></a></h3>
+<p>Make sure the <code>maven-deploy-plugin</code> is correctly configured in your <code>pom.xml</code>. Here's an example configuration:</p>
+<div class="language-xml codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-xml codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token tag punctuation" style="color:#393A34"><</span><span class="token tag" style="color:#00009f">plug [...]
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="7-increase-verbose-logging">7. Increase Verbose Logging<a href="https://wayang.apache.org/blog/wayang-release-odysse#7-increase-verbose-logging" class="hash-link" aria-label="Direct link to 7. Increase Verbose Logging" title="Direct link to 7. Increase Verbose Logging"></a></h3>
+<p>Enable verbose logging to get more details about the error. You can do this by adding the <code>-X</code> flag when running the Maven command:</p>
+<div class="language-sh codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-sh codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn clean deploy </span><span class="token parameter variable" style="color:#36acaa">-X</span>< [...]
+<p><strong>[DONE]</strong></p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="8-retry-with-a-clean-local-repository">8. Retry with a Clean Local Repository<a href="https://wayang.apache.org/blog/wayang-release-odysse#8-retry-with-a-clean-local-repository" class="hash-link" aria-label="Direct link to 8. Retry with a Clean Local Repository" title="Direct link to 8. Retry with a Clean Local Repository"></a></h3>
+<p>Sometimes, a corrupt local repository can cause issues. Try cleaning your local Maven repository and re-running the deployment:</p>
+<div class="language-sh codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-sh codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn clean </span><span class="token function" style="color:#d73a49">install</span><span class=" [...]
+<p><em>[OPEN]</em> - Does not match to an authentication issue.</p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="9-check-for-missing-credentials">9. Check for Missing Credentials<a href="https://wayang.apache.org/blog/wayang-release-odysse#9-check-for-missing-credentials" class="hash-link" aria-label="Direct link to 9. Check for Missing Credentials" title="Direct link to 9. Check for Missing Credentials"></a></h3>
+<p>Ensure that the credentials for the repository are correctly set up in your <code>settings.xml</code>:</p>
+<div class="language-xml codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-xml codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token tag punctuation" style="color:#393A34"><</span><span class="token tag" style="color:#00009f">serv [...]
+<p><strong>[DONE]</strong></p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="10-review-the-full-stack-trace">10. Review the Full Stack Trace<a href="https://wayang.apache.org/blog/wayang-release-odysse#10-review-the-full-stack-trace" class="hash-link" aria-label="Direct link to 10. Review the Full Stack Trace" title="Direct link to 10. Review the Full Stack Trace"></a></h3>
+<p>The full stack trace indicates a <code>NullPointerException</code>:</p>
+<div class="language-plaintext codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-plaintext codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">Caused by: java.lang.NullPointerException</span><br></span><span class="token-lin [...]
+<p>This suggests that there might be an issue with how credentials are being handled. Double-check that the credentials are being correctly passed and processed.</p>
+<p><strong>[DONE]</strong></p>]]></content:encoded>
<category>wayang</category>
<category>ASF</category>
<category>release</category>
@@ -453,7 +339,7 @@ expect to see soon.</p>
<category>python</category>
</item>
<item>
- <title><![CDATA[Apache Kafka meets Apache Wayang - Part 3]]></title>
+ <title><![CDATA[Apache Kafka meets Wayang - Part 3]]></title>
<link>https://wayang.apache.org/blog/kafka-meets-wayang-3</link>
<guid>https://wayang.apache.org/blog/kafka-meets-wayang-3</guid>
<pubDate>Sun, 10 Mar 2024 00:00:00 GMT</pubDate>
@@ -579,7 +465,7 @@ In fact, Trino can be easily plugged to Wayang as a platform and be seamlessly i
<category>trino</category>
</item>
<item>
- <title><![CDATA[Apache Kafka meets Apache Wayang - Part 2]]></title>
+ <title><![CDATA[Apache Kafka meets Wayang - Part 2]]></title>
<link>https://wayang.apache.org/blog/kafka-meets-wayang-2</link>
<guid>https://wayang.apache.org/blog/kafka-meets-wayang-2</guid>
<pubDate>Wed, 06 Mar 2024 00:00:00 GMT</pubDate>
@@ -608,8 +494,8 @@ We created the Kafka Source and Kafka Sink components so that our cross organiza
<p><strong>Level 1 – Wayang execution plan with abstract operators</strong></p>
<p>The implementation of our Kafka Source and Kafka Sink components for Apache Wayang requires new methods and classes on three layers.
First of all in the API package.
-Here we use the JavaPlanBuilder to expose the function for selecting a Kafka topic as the source to be used by client.<br>
-<!-- -->The class <em>JavaPlanBuilder</em> in package <em>org.apache.wayang.api</em> in the project <em>wayang-api/wayang-api-scala-java</em> exposes our new functionality to our external client.
+Here we use the JavaPlanBuilder to expose the function for selecting a Kafka topic as the source to be used by client.
+The class <em>JavaPlanBuilder</em> in package <em>org.apache.wayang.api</em> in the project <em>wayang-api/wayang-api-scala-java</em> exposes our new functionality to our external client.
An instance of the JavaPlanBuilder is used to define the data processing pipeline.
We use its <em>readKafkaTopic()</em> which specifies the source Kafka topic to read from, and for the write path we use the <em>writeKafkaTopic()</em> method.
Both Methods do only trigger activities in the background.</p>
@@ -624,8 +510,8 @@ In this API layer we use the Scala programming language, but we utilize the Java
<p>Therefore, the mappings package has a class <em>Mappings</em> in which all relevant input and output operators are listed.
We use it to register the KafkaSourceMapping and a KafkaSinkMapping for the particular platform, Java in our case.
These classes allow the Apache Wayang framework to use the Java implementation of the KafkaTopicSource component (and KafkaTopicSink respectively).
-While the Wayang execution plan uses the higher abstractions, here on the “platform level” we have to link the specific implementation for the target platform.</p>
-<p>In our case this leads to a Java program running on a JVM which is set up by the Apache Wayang framework using the logical components of the execution plan.</p>
+While the Wayang execution plan uses the higher abstractions, here on the “platform level” we have to link the specific implementation for the target platform.
+In our case this leads to a Java program running on a JVM which is set up by the Apache Wayang framework using the logical components of the execution plan.</p>
<p>Those mappings link the real implementation of our operators the ones used in an execution plan.
The JavaKafkaTopicSource and the JavaKafkaTopicSink extend the KafkaTopicSource and KafkaTopicSink so that the lower level implementation of those classes become available within Wayang’s Java Platform context.</p>
<p>In this layer, the KafkaConsumer class and the KafkaProducer class are used, but both are configured and instantiated in the next layer underneath.
@@ -640,14 +526,14 @@ Details about consumer and producers, client configuration, and schema handling
The layer above handles the mapping of the components at startup time.
All this wiring is needed to keep Wayang open and flexible so that multiple external systems can be used in a variety of combinations and using multiple target platforms in combinations.</p>
<h2 class="anchor anchorWithStickyNavbar_LWe7" id="outlook">Outlook<a href="https://wayang.apache.org/blog/kafka-meets-wayang-2#outlook" class="hash-link" aria-label="Direct link to Outlook" title="Direct link to Outlook"></a></h2>
-<p>The next part of the article series will cover the creation of an Kafka Source and Sink component for the Apache Spark platform, which allows our use case to scale.</p>
-<p>Finally, in part four we bring all puzzles together, and show the full implementation of the multi organizational data collaboration use case.</p>]]></content:encoded>
+<p>The next part of the article series will cover the creation of an Kafka Source and Sink component for the Apache Spark platform, which allows our use case to scale.
+Finally, in part four we bring all puzzles together, and show the full implementation of the multi organizational data collaboration use case.</p>]]></content:encoded>
<category>wayang</category>
<category>kafka</category>
<category>cross organization data collaboration</category>
</item>
<item>
- <title><![CDATA[Apache Kafka meets Apache Wayang - Part 1]]></title>
+ <title><![CDATA[Apache Kafka meets Wayang - Part 1]]></title>
<link>https://wayang.apache.org/blog/kafka-meets-wayang-1</link>
<guid>https://wayang.apache.org/blog/kafka-meets-wayang-1</guid>
<pubDate>Tue, 05 Mar 2024 00:00:00 GMT</pubDate>
@@ -657,9 +543,9 @@ All this wiring is needed to keep Wayang open and flexible so that multiple exte
The first article starts with an introduction of a typical data colaboration scenario which will emerge in our digital future.</p>
<p>In part two and three we will share a summary of our Apache Kafka client implementation for Apache Wayang.
We started with the Java Platform (part 2) and the Apache Spark implementation follows (W.I.P.) in part three.</p>
-<p>The use case behind this work is an imaginary data collaboration scenario.</p>
-<p>We see this example and the demand for a solution already in many places.</p>
-<p>For us this is motivation enough to propose a solution.
+<p>The use case behind this work is an imaginary data collaboration scenario.
+We see this example and the demand for a solution already in many places.
+For us this is motivation enough to propose a solution.
This would also allow us to do more local data processing, and businesses can stop moving data around the world, but rather care about data locality while they expose and share specific information to others by using data federation.
This reduces complexity of data management and cost dramatically.</p>
<p>For this purpose, we illustrate a cross organizational data sharing scenario from the finance sector soon.
@@ -694,8 +580,8 @@ But they share some of the data, using publicly accessible Kafka topics, marked
Since as of today, there is now <em>KafkaSource</em> and <em>KafkaSink</em> available in Apache Wayang, an implementation of both will be our first step.
Our assumption is, that in the beginning, there won’t be much data.</p>
<p>Apache Spark is not required to cope with the load, but we expect, that in the future, a single Java application would not be able to handle our workload.
-Hence, we want to utilize the Apache Wayang abstraction over multiple processing platforms, starting with Java.</p>
-<p>Later, we want to switch to Apache Spark.</p>]]></content:encoded>
+Hence, we want to utilize the Apache Wayang abstraction over multiple processing platforms, starting with Java.
+Later, we want to switch to Apache Spark.</p>]]></content:encoded>
<category>wayang</category>
<category>kafka</category>
<category>cross organization data collaboration</category>
diff --git a/blog/tags/ai/index.html b/blog/tags/ai/index.html
index 6992d6ce..736e340e 100644
--- a/blog/tags/ai/index.html
+++ b/blog/tags/ai/index.html
@@ -5,11 +5,11 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">One post tagged with "ai" | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/blog/tags/ai"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:langua [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.01bb8940.js" defer="defer"></script>
-<script src="/assets/js/main.0cc2e93a.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.8489d39d.js" defer="defer"></script>
+<script src="/assets/js/main.4f5176d9.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
-<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
+<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
detecting early signs of cancer cells. Building an AI requires a lot of data and training to achieve
the desired results, and federated learning is an approach to make AI training more viable.
Federated learning (or collaborative learning) is a technique that trains AI models on data
diff --git a/blog/tags/asf/index.html b/blog/tags/asf/index.html
index 37d06472..2643b242 100644
--- a/blog/tags/asf/index.html
+++ b/blog/tags/asf/index.html
@@ -5,54 +5,131 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">One post tagged with "ASF" | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/blog/tags/asf"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:lang [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.01bb8940.js" defer="defer"></script>
-<script src="/assets/js/main.0cc2e93a.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.8489d39d.js" defer="defer"></script>
+<script src="/assets/js/main.4f5176d9.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
-<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
-<p>This article is the first of a four part series about federated data analysis using Apache Wayang.
-The first article starts with an introduction of a typical data colaboration scenario which will emerge in our digital future.</p>
-<p>In part two and three we will share a summary of our Apache Kafka client implementation for Apache Wayang.
-We started with the Java Platform (part 2) and the Apache Spark implementation follows (W.I.P.) in part three.</p>
-<p>The use case behind this work is an imaginary data collaboration scenario.
-We see this example and the demand for a solution already in many places.
-For us this is motivation enough to propose a solution.
-This would also allow us to do more local data processing, and businesses can stop moving data around the world, but rather care about data locality while they expose and share specific information to others by using data federation.
-This reduces complexity of data management and cost dramatically.</p>
-<p>For this purpose, we illustrate a cross organizational data sharing scenario from the finance sector soon.
-This analysis pattern will also be relevant in the context of data analysis along supply chains, another typical example where data from many stakeholder together is needed but never managed in one place, for good reasons.</p>
-<p>Data federation can help us to unlock the hidden value of all those isolated data lakes.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="a-cross-organizational-data-sharing-scenario">A cross organizational data sharing scenario<a href="#a-cross-organizational-data-sharing-scenario" class="hash-link" aria-label="Direct link to A cross organizational data sharing scenario" title="Direct link to A cross organizational data sharing scenario"></a></h2>
-<p>Our goal is the implementation of a cross organization decentralized data processing scenario, in which protected local data should be processed in combination with public data from public sources in a collaborative manner.
-Instead of copying all data into a central data lake or a central data platform we decided to use federated analytics.
-Apache Wayang is the tool we work with.
-In our case, the public data is hosted on publicly available websites or data pods.
-A client can use the HTTP(S) protocol to read the data which is given in a well defined format.
-For simplicity we decided to use CSV format.
-When we look into the data of each participant we have a different perspective.</p>
-<p>Our processing procedure should calculate a particular metric on the <em>local data</em> of each participant.
-An example of such a metric is the average spending of all users on a particular product category per month.
-This can vary from partner to partner, hence, we want to be able to calculate a peer-group comparison so that each partner can see its own metric compared with a global average calculated from contributions by all partners.
-Such a process requires global averaging and local averaging.
-And due to governance constraints, we can’t bring all raw data together in one place.</p>
-<p>Instead, we want to use Apache Wayang for this purpose.
-We simplify the procedure and split it into two phases.
-Phase one is the process, which allows each participant to calculate the local metrics.
-This requires only local data. The second phase requires data from all collaborating partners.
-The monthly sum and counter values per partner and category are needed in one place by all other parties.
-Hence, the algorithm of the first phase stores the local results locally, and the contributions to the global results in an externally accessible Kafka topic.
-We assume this is done by each of the partners.</p>
-<p>Now we have a scenario, in which an Apache Wayang process must be able to read data from multiple Apache Kafka topics from multiple Apache Kafka clusters but finally writes into a single Kafka topic, which then can be accessed by all the participating clients.</p>
-<p><img decoding="async" loading="lazy" alt="images/image-1.png" src="/assets/images/image-1-9cc35d5aea2b867d7e5759a96bd02334.png" width="904" height="550" class="img_ev3q"></p>
-<p>The illustration shows the data flows in such a scenario.
-Jobs with red border are executed by the participants in isolation within their own data processing environments.
-But they share some of the data, using publicly accessible Kafka topics, marked by A. Job 4 is the Apache Wayang job in our focus: here we intent to read data from 3 different source systems, and write results into a fourth system (marked as B), which can be accesses by all participants again.</p>
-<p>With this in mind we want to implement an Apache Wayang application which implements the illustrated <em>Job 4</em>.
-Since as of today, there is now <em>KafkaSource</em> and <em>KafkaSink</em> available in Apache Wayang, an implementation of both will be our first step.
-Our assumption is, that in the beginning, there won’t be much data.</p>
-<p>Apache Spark is not required to cope with the load, but we expect, that in the future, a single Java application would not be able to handle our workload.
-Hence, we want to utilize the Apache Wayang abstraction over multiple processing platforms, starting with Java.
-Later, we want to switch to Apache Spark.</p></div><footer class="row docusaurus-mt-lg"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/kafka">kafka</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/cross-organization-data-collaboration">cross organization [...]
+<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
+<p>The ASF provides a robust infrastructure for open communities of software developers. We can share ideas, combine forces, contribute code, docs, review-energy, art work, and from time to time we can nail it down. A release defines an intermediate result of the continuous community work.</p>
+<p>How we do such a release in the Apache Wayang team is an essential aspect towards graduation. First of all, there are some references to take into account, such as:</p>
+<ul>
+<li><a href="https://www.apache.org/legal/release-policy.html" target="_blank" rel="noopener noreferrer">https://www.apache.org/legal/release-policy.html</a></li>
+<li><a href="https://maven.apache.org/maven-release/maven-release-plugin/" target="_blank" rel="noopener noreferrer">https://maven.apache.org/maven-release/maven-release-plugin/</a></li>
+</ul>
+<p>Assuming you are (P)PMC, and assuming that you have the right permissions for such a release, you can follow the path as described in this guide:</p>
+<ul>
+<li><a href="https://plc4x.apache.org/developers/release/release.html" target="_blank" rel="noopener noreferrer">https://plc4x.apache.org/developers/release/release.html</a></li>
+</ul>
+<p>I tried to follow exactly this procedure, several times. I failed. Here I share the current status of my <strong>release attempts</strong>.</p>
+<p>I plan a longer tour, and do not want to block the project for a long time.
+Hence I create this draft, and I hope we can unblock this project as soon as possible.</p>
+<h2 class="anchor anchorWithStickyNavbar_LWe7" id="status">Status:<a href="#status" class="hash-link" aria-label="Direct link to Status:" title="Direct link to Status:"></a></h2>
+<p>I am not able to conduct the <em>mvn release:perform</em> step.
+Anything before worked, sometimes only after some digging, but it worked.</p>
+<ul>
+<li>
+<p>We assume, that due to my membership in two ASF incubator projects I am not able to upload the artefacts to the Nexus repository (H1).</p>
+</li>
+<li>
+<p>It can be, that I have not the correct user and password in my <em>settings.xml</em> file (H2).</p>
+</li>
+</ul>
+<p>But I tested a manual login to the nexus server <a href="https://repository.apache.org/service/local/staging/deploy/maven2" target="_blank" rel="noopener noreferrer">https://repository.apache.org/service/local/staging/deploy/maven2</a> with success. And beyond that I have no idea how I can verify this detail alone.</p>
+<h2 class="anchor anchorWithStickyNavbar_LWe7" id="idea--proposal">Idea / Proposal<a href="#idea--proposal" class="hash-link" aria-label="Direct link to Idea / Proposal" title="Direct link to Idea / Proposal"></a></h2>
+<p>(1) It would be great, if someone - who has done a release in any other ASF project or in Apache Wayang - could follow the steps I share, so that we can check where the problem hides itself.</p>
+<p>(2) As a follow-up task, I suggest to add a <strong>Release Guide</strong> to the Apache Wayang project, including release manager onboarding steps, and checklists for the particular project, derived from the referenced sources which are listed above.</p>
+<p>But for now it is all about sharing the status (as I did serveral times on multiple chanels, including JIRA, Slack, Mailing lists) and finding a solution for Apache Wayang release 1.0.</p>
+<h2 class="anchor anchorWithStickyNavbar_LWe7" id="latest-error">Latest Error:<a href="#latest-error" class="hash-link" aria-label="Direct link to Latest Error:" title="Direct link to Latest Error:"></a></h2>
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn release:perform -X -DskipTests</span><br></span></code></pre><div class="buttonGroup__atx"><button ty [...]
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">[INFO] Caused by: org.eclipse.aether.deployment.DeploymentException: Failed to deploy artifacts: Could no [...]
+<h2 class="anchor anchorWithStickyNavbar_LWe7" id="activity-log">Activity Log<a href="#activity-log" class="hash-link" aria-label="Direct link to Activity Log" title="Direct link to Activity Log"></a></h2>
+<div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn release:clean</span><br></span><span class="token-line" style="color:#393A34"><span cla [...]
+<div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn release:prepare </span><span class="token parameter variable" style="color:#36acaa">-Da [...]
+<div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token punctuation" style="color:#393A34">[</span><span class="token plain">ERROR</span><span class="toke [...]
+<div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn versions:set </span><span class="token parameter variable" style="color:#36acaa">-DnewV [...]
+<div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn release:prepare </span><span class="token parameter variable" style="color:#36acaa">-Da [...]
+<div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token punctuation" style="color:#393A34">[</span><span class="token plain">ERROR</span><span class="toke [...]
+<div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token function" style="color:#d73a49">git</span><span class="token plain"> status</span><br></span><span [...]
+<div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn release:prepare </span><span class="token parameter variable" style="color:#36acaa">-Da [...]
+<div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">Caused by: org.eclipse.aether.transfer.NoRepositoryConnectorException: Blocked mirror </spa [...]
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="dependency-on-jdk-11-during-release">Dependency on JDK-11 during release<a href="#dependency-on-jdk-11-during-release" class="hash-link" aria-label="Direct link to Dependency on JDK-11 during release" title="Direct link to Dependency on JDK-11 during release"></a></h3>
+<blockquote>
+<p>FIXED with local JDK11 Setup</p>
+</blockquote>
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">brew install openjdk@11</span><br></span><span class="token-line" style="color:#393A34"><span class="toke [...]
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="manual-update-of-release-version">Manual update of release-version<a href="#manual-update-of-release-version" class="hash-link" aria-label="Direct link to Manual update of release-version" title="Direct link to Manual update of release-version"></a></h3>
+<p>During the release procedure, do I have to set the version here in this configuration section manually?</p>
+<div class="language-xml codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-xml codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token tag punctuation" style="color:#393A34"><</span><span class="token tag" style="color:#00009f">plug [...]
+<blockquote>
+<p>It seems that these properties must be updated manually.</p>
+</blockquote>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="warning-regarding-illegal-reflective-access-operation">Warning regarding "illegal reflective access operation"<a href="#warning-regarding-illegal-reflective-access-operation" class="hash-link" aria-label="Direct link to Warning regarding "illegal reflective access operation"" title="Direct link to Warning regarding "illegal reflective access operation""></a></h3>
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">[ERROR] WARNING: An illegal reflective access operation has occurred</span><br></span><span class="token- [...]
+<blockquote>
+<p><strong>This is still an OPEN ISSUE !</strong></p>
+</blockquote>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="rat-check-fails">RAT Check fails<a href="#rat-check-fails" class="hash-link" aria-label="Direct link to RAT Check fails" title="Direct link to RAT Check fails"></a></h3>
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">[INFO] [ERROR] Failed to execute goal org.apache.rat:apache-rat-plugin:0.13:check (license-check) on proj [...]
+<div class="language-cat codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-cat codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#393A34" [...]
+<blockquote>
+<p><em>FIXED by adding .java-versions to .gitignore</em></p>
+</blockquote>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="tag-could-not-be-created-in-scm">Tag could not be created in SCM.<a href="#tag-could-not-be-created-in-scm" class="hash-link" aria-label="Direct link to Tag could not be created in SCM." title="Direct link to Tag could not be created in SCM."></a></h3>
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">[ERROR] Failed to execute goal org.apache.maven.plugins:maven-release-plugin:3.0.1:prepare (default-cli) [...]
+<blockquote>
+<p>FIXED by manual changes in pom.xml.</p>
+</blockquote>
+<div class="language-xml codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-xml codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#393A34" [...]
+<div class="language-xml codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-xml codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token tag punctuation" style="color:#393A34"><</span><span class="token tag" style="color:#00009f">conf [...]
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn release:prepare -Darguments='-DskipTests=True -Dresume=True'</span><br></span></code></pre> [...]
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn clean package</span><br></span><span class="token-line" style="color:#393A34"><span class="token plai [...]
+<blockquote>
+<p>So far so good. But now the sun went down.</p>
+</blockquote>
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">[INFO] [ERROR] Failed to execute goal org.apache.maven.plugins:maven-deploy-plugin:3.0.0-M1:deploy (defau [...]
+<p>The null pointer exception indicates problems in <em>settings.xml</em>. This could be fixed.
+But now, the error changes with and I am still not able to stage the build results.</p>
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn release:perform -X -DskipTests</span><br></span></code></pre><div class="buttonGroup__atx"><button ty [...]
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">[INFO] Caused by: org.eclipse.aether.deployment.DeploymentException: Failed to deploy artifacts: Could no [...]
+<h2 class="anchor anchorWithStickyNavbar_LWe7" id="checklist-for-next-iteration">Checklist for next iteration:<a href="#checklist-for-next-iteration" class="hash-link" aria-label="Direct link to Checklist for next iteration:" title="Direct link to Checklist for next iteration:"></a></h2>
+<p>The error you're encountering indicates that there is a problem with deploying the artifacts using the <code>maven-deploy-plugin</code>. The root cause of the error is a <code>NullPointerException</code> during the deployment process, specifically related to the <code>org.eclipse.aether.transfer.ArtifactTransferException</code>.</p>
+<p>Here's a step-by-step guide to troubleshoot and resolve this issue:</p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="1-check-maven-settings">1. Check Maven Settings<a href="#1-check-maven-settings" class="hash-link" aria-label="Direct link to 1. Check Maven Settings" title="Direct link to 1. Check Maven Settings"></a></h3>
+<p>Ensure that your Maven settings (<code>settings.xml</code>) are correctly configured for deployment. Verify that the repository settings and credentials are correctly specified.</p>
+<p><strong>[DONE]</strong></p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="2-verify-repository-url">2. Verify Repository URL<a href="#2-verify-repository-url" class="hash-link" aria-label="Direct link to 2. Verify Repository URL" title="Direct link to 2. Verify Repository URL"></a></h3>
+<p>Make sure the repository URL in your <code>pom.xml</code> or <code>settings.xml</code> is correct and reachable. The URL should point to the correct staging repository for deployment.</p>
+<p><em>[OPEN]</em> - I did not touch it, so I guess it is correct in the <em>pom.xml</em>.</p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="3-maven-version-compatibility">3. Maven Version Compatibility<a href="#3-maven-version-compatibility" class="hash-link" aria-label="Direct link to 3. Maven Version Compatibility" title="Direct link to 3. Maven Version Compatibility"></a></h3>
+<p>Verify that you are using a compatible version of Maven. Sometimes, upgrading or downgrading Maven can resolve such issues.</p>
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">Apache Maven 3.9.6 (bc0240f3c744dd6b6ec2920b3cd08dcc295161ae)</span><br></span><span class="token-line" s [...]
+<p><strong>[DONE]</strong></p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="4-check-for-network-issues">4. Check for Network Issues<a href="#4-check-for-network-issues" class="hash-link" aria-label="Direct link to 4. Check for Network Issues" title="Direct link to 4. Check for Network Issues"></a></h3>
+<p>Ensure there are no network issues that might be causing problems in connecting to the repository. Sometimes, network configurations, firewalls, or proxy settings can interfere with the deployment process.</p>
+<p><strong>[DONE]</strong></p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="5-update-maven-plugins">5. Update Maven Plugins<a href="#5-update-maven-plugins" class="hash-link" aria-label="Direct link to 5. Update Maven Plugins" title="Direct link to 5. Update Maven Plugins"></a></h3>
+<p>Ensure that you are using the latest versions of the Maven plugins. Sometimes, bugs in older versions can cause unexpected issues.</p>
+<p><em>[OPEN]</em></p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="6-configure-the-maven-deploy-plugin-in-pomxml">6. Configure the <code>maven-deploy-plugin</code> in <code>pom.xml</code><a href="#6-configure-the-maven-deploy-plugin-in-pomxml" class="hash-link" aria-label="Direct link to 6-configure-the-maven-deploy-plugin-in-pomxml" title="Direct link to 6-configure-the-maven-deploy-plugin-in-pomxml"></a></h3>
+<p>Make sure the <code>maven-deploy-plugin</code> is correctly configured in your <code>pom.xml</code>. Here's an example configuration:</p>
+<div class="language-xml codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-xml codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token tag punctuation" style="color:#393A34"><</span><span class="token tag" style="color:#00009f">plug [...]
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="7-increase-verbose-logging">7. Increase Verbose Logging<a href="#7-increase-verbose-logging" class="hash-link" aria-label="Direct link to 7. Increase Verbose Logging" title="Direct link to 7. Increase Verbose Logging"></a></h3>
+<p>Enable verbose logging to get more details about the error. You can do this by adding the <code>-X</code> flag when running the Maven command:</p>
+<div class="language-sh codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-sh codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn clean deploy </span><span class="token parameter variable" style="color:#36acaa">-X</span>< [...]
+<p><strong>[DONE]</strong></p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="8-retry-with-a-clean-local-repository">8. Retry with a Clean Local Repository<a href="#8-retry-with-a-clean-local-repository" class="hash-link" aria-label="Direct link to 8. Retry with a Clean Local Repository" title="Direct link to 8. Retry with a Clean Local Repository"></a></h3>
+<p>Sometimes, a corrupt local repository can cause issues. Try cleaning your local Maven repository and re-running the deployment:</p>
+<div class="language-sh codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-sh codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn clean </span><span class="token function" style="color:#d73a49">install</span><span class=" [...]
+<p><em>[OPEN]</em> - Does not match to an authentication issue.</p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="9-check-for-missing-credentials">9. Check for Missing Credentials<a href="#9-check-for-missing-credentials" class="hash-link" aria-label="Direct link to 9. Check for Missing Credentials" title="Direct link to 9. Check for Missing Credentials"></a></h3>
+<p>Ensure that the credentials for the repository are correctly set up in your <code>settings.xml</code>:</p>
+<div class="language-xml codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-xml codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token tag punctuation" style="color:#393A34"><</span><span class="token tag" style="color:#00009f">serv [...]
+<p><strong>[DONE]</strong></p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="10-review-the-full-stack-trace">10. Review the Full Stack Trace<a href="#10-review-the-full-stack-trace" class="hash-link" aria-label="Direct link to 10. Review the Full Stack Trace" title="Direct link to 10. Review the Full Stack Trace"></a></h3>
+<p>The full stack trace indicates a <code>NullPointerException</code>:</p>
+<div class="language-plaintext codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-plaintext codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">Caused by: java.lang.NullPointerException</span><br></span><span class="token-lin [...]
+<p>This suggests that there might be an issue with how credentials are being handled. Double-check that the credentials are being correctly passed and processed.</p>
+<p><strong>[DONE]</strong></p></div><footer class="row docusaurus-mt-lg"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/asf">ASF</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/release">release</a></li></ul></div></footer></article><nav class="paginatio [...]
<p> Apache Wayang is an effort undergoing incubation at The Apache Software Foundation (ASF), sponsored by the Apache Incubator. Incubation is required of all newly accepted projects until a further review indicates that the infrastructure, communications, and decision making process have stabilized in a manner consistent with other successful ASF projects. While incubation status is not necessarily a reflection of the completeness or stability of the code, it does indicate that th [...]
<p>
Copyright © 2024 The Apache Software Foundation, Licensed under the Apache License, Version 2.0. <br>
diff --git a/blog/tags/cross-organization-data-collaboration/index.html b/blog/tags/cross-organization-data-collaboration/index.html
index 4ec2a3ea..2e866817 100644
--- a/blog/tags/cross-organization-data-collaboration/index.html
+++ b/blog/tags/cross-organization-data-collaboration/index.html
@@ -3,170 +3,13 @@
<head>
<meta charset="UTF-8">
<meta name="generator" content="Docusaurus v3.1.1">
-<title data-rh="true">6 posts tagged with "cross organization data collaboration" | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/blog/tags/cross-organization-data-collaboration"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusa [...]
+<title data-rh="true">3 posts tagged with "cross organization data collaboration" | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/blog/tags/cross-organization-data-collaboration"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusa [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.01bb8940.js" defer="defer"></script>
-<script src="/assets/js/main.0cc2e93a.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.8489d39d.js" defer="defer"></script>
+<script src="/assets/js/main.4f5176d9.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
-<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
-<p>This article is the first of a four part series about federated data analysis using Apache Wayang.
-The first article starts with an introduction of a typical data colaboration scenario which will emerge in our digital future.</p>
-<p>In part two and three we will share a summary of our Apache Kafka client implementation for Apache Wayang.
-We started with the Java Platform (part 2) and the Apache Spark implementation follows (W.I.P.) in part three.</p>
-<p>The use case behind this work is an imaginary data collaboration scenario.
-We see this example and the demand for a solution already in many places.
-For us this is motivation enough to propose a solution.
-This would also allow us to do more local data processing, and businesses can stop moving data around the world, but rather care about data locality while they expose and share specific information to others by using data federation.
-This reduces complexity of data management and cost dramatically.</p>
-<p>For this purpose, we illustrate a cross organizational data sharing scenario from the finance sector soon.
-This analysis pattern will also be relevant in the context of data analysis along supply chains, another typical example where data from many stakeholder together is needed but never managed in one place, for good reasons.</p>
-<p>Data federation can help us to unlock the hidden value of all those isolated data lakes.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="a-cross-organizational-data-sharing-scenario">A cross organizational data sharing scenario<a href="#a-cross-organizational-data-sharing-scenario" class="hash-link" aria-label="Direct link to A cross organizational data sharing scenario" title="Direct link to A cross organizational data sharing scenario"></a></h2>
-<p>Our goal is the implementation of a cross organization decentralized data processing scenario, in which protected local data should be processed in combination with public data from public sources in a collaborative manner.
-Instead of copying all data into a central data lake or a central data platform we decided to use federated analytics.
-Apache Wayang is the tool we work with.
-In our case, the public data is hosted on publicly available websites or data pods.
-A client can use the HTTP(S) protocol to read the data which is given in a well defined format.
-For simplicity we decided to use CSV format.
-When we look into the data of each participant we have a different perspective.</p>
-<p>Our processing procedure should calculate a particular metric on the <em>local data</em> of each participant.
-An example of such a metric is the average spending of all users on a particular product category per month.
-This can vary from partner to partner, hence, we want to be able to calculate a peer-group comparison so that each partner can see its own metric compared with a global average calculated from contributions by all partners.
-Such a process requires global averaging and local averaging.
-And due to governance constraints, we can’t bring all raw data together in one place.</p>
-<p>Instead, we want to use Apache Wayang for this purpose.
-We simplify the procedure and split it into two phases.
-Phase one is the process, which allows each participant to calculate the local metrics.
-This requires only local data. The second phase requires data from all collaborating partners.
-The monthly sum and counter values per partner and category are needed in one place by all other parties.
-Hence, the algorithm of the first phase stores the local results locally, and the contributions to the global results in an externally accessible Kafka topic.
-We assume this is done by each of the partners.</p>
-<p>Now we have a scenario, in which an Apache Wayang process must be able to read data from multiple Apache Kafka topics from multiple Apache Kafka clusters but finally writes into a single Kafka topic, which then can be accessed by all the participating clients.</p>
-<p><img decoding="async" loading="lazy" alt="images/image-1.png" src="/assets/images/image-1-9cc35d5aea2b867d7e5759a96bd02334.png" width="904" height="550" class="img_ev3q"></p>
-<p>The illustration shows the data flows in such a scenario.
-Jobs with red border are executed by the participants in isolation within their own data processing environments.
-But they share some of the data, using publicly accessible Kafka topics, marked by A. Job 4 is the Apache Wayang job in our focus: here we intent to read data from 3 different source systems, and write results into a fourth system (marked as B), which can be accesses by all participants again.</p>
-<p>With this in mind we want to implement an Apache Wayang application which implements the illustrated <em>Job 4</em>.
-Since as of today, there is now <em>KafkaSource</em> and <em>KafkaSink</em> available in Apache Wayang, an implementation of both will be our first step.
-Our assumption is, that in the beginning, there won’t be much data.</p>
-<p>Apache Spark is not required to cope with the load, but we expect, that in the future, a single Java application would not be able to handle our workload.
-Hence, we want to utilize the Apache Wayang abstraction over multiple processing platforms, starting with Java.
-Later, we want to switch to Apache Spark.</p></div><footer class="row docusaurus-mt-lg"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/kafka">kafka</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/cross-organization-data-collaboration">cross organization [...]
-We look into the “Read- and Write-Path” for our data items, called <em>DataQuanta</em>.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="apache-wayangs-read--write-path-for-kafka-topics">Apache Wayang’s Read & Write Path for Kafka topics<a href="#apache-wayangs-read--write-path-for-kafka-topics" class="hash-link" aria-label="Direct link to Apache Wayang’s Read & Write Path for Kafka topics" title="Direct link to Apache Wayang’s Read & Write Path for Kafka topics"></a></h2>
-<p>To describe the read and write paths for data in the context of the created Apache Wayang code snippet, the primary classes and interfaces we need to understand are as follows:</p>
-<p><strong>WayangContext:</strong> This class is essential for initializing the Wayang processing environment.
-It allows you to configure the execution environment and register plugins that define which platforms Wayang can use for data processing tasks, such as <em>Java.basicPlugin()</em> for local Java execution.</p>
-<p><strong>JavaPlanBuilder:</strong> This class is used to build and define the data processing pipeline (or plan) in Wayang.
-It provides a fluent API to specify the operations to be performed on the data, from reading the input to processing it and writing the output.</p>
-<h3 class="anchor anchorWithStickyNavbar_LWe7" id="read-path">Read Path<a href="#read-path" class="hash-link" aria-label="Direct link to Read Path" title="Direct link to Read Path"></a></h3>
-<p>The read path describes how data is ingested from a source into the Wayang processing pipeline:</p>
-<p><em>Reading from Kafka Topic:</em> The method <em>readKafkaTopic(topicName)</em> is used to ingest data from a specified Kafka topic.
-This is the starting point of the data processing pipeline, where topicName represents the name of the Kafka topic from which data is read.</p>
-<p><em>Data Tokenization and Preparation:</em> Once the data is read from Kafka, it undergoes several transformations such as Splitting, Filtering, and Mapping.
-What follows are the procedures known as Reducing, Grouping, Co-Grouping, and Counting.</p>
-<h3 class="anchor anchorWithStickyNavbar_LWe7" id="write-path">Write Path<a href="#write-path" class="hash-link" aria-label="Direct link to Write Path" title="Direct link to Write Path"></a></h3>
-<p><em>Writing to Kafka Topic:</em> The final step in the pipeline involves writing the processed data back to a Kafka topic using <em>.writeKafkaTopic(...)</em>.
-This method takes parameters that specify the target Kafka topic, a serialization function to format the data as strings, and additional configuration for load profile estimation, which optimizes the writing process.</p>
-<p>This read-write path provides a comprehensive flow of data from ingestion from Kafka, through various processing steps, and finally back to Kafka, showcasing a full cycle of data processing within Apache Wayang's abstracted environment and is implemented in our example program shown in <em>listing 1</em>.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="implementation-of-input--and-output-operators">Implementation of Input- and Output Operators<a href="#implementation-of-input--and-output-operators" class="hash-link" aria-label="Direct link to Implementation of Input- and Output Operators" title="Direct link to Implementation of Input- and Output Operators"></a></h2>
-<p>The next section shows how a new pair of operators can be implemented to extend Apache Wayang’s capabilities on the input and output side.
-We created the Kafka Source and Kafka Sink components so that our cross organizational data collaboration scenario can be implemented using data streaming infrastructure.</p>
-<p><strong>Level 1 – Wayang execution plan with abstract operators</strong></p>
-<p>The implementation of our Kafka Source and Kafka Sink components for Apache Wayang requires new methods and classes on three layers.
-First of all in the API package.
-Here we use the JavaPlanBuilder to expose the function for selecting a Kafka topic as the source to be used by client.
-The class <em>JavaPlanBuilder</em> in package <em>org.apache.wayang.api</em> in the project <em>wayang-api/wayang-api-scala-java</em> exposes our new functionality to our external client.
-An instance of the JavaPlanBuilder is used to define the data processing pipeline.
-We use its <em>readKafkaTopic()</em> which specifies the source Kafka topic to read from, and for the write path we use the <em>writeKafkaTopic()</em> method.
-Both Methods do only trigger activities in the background.</p>
-<p>For the output side, we use the <em>DataQuantaBuilder</em> class, which offers an implementation of the writeKafkaTopic function.
-This function is designed to send processed data, referred to as DataQuanta, to a specified Kafka topic.
-Essentially, it marks the final step in a data processing sequence constructed using the Apache Wayang framework.</p>
-<p>In the DataQuanta class we implemented the methods writeKafkaTopic and writeKafkaTopicJava which use the KafkaTopicSink class.
-In this API layer we use the Scala programming language, but we utilize the Java classes, implemented in the layer below.</p>
-<p><strong>Level 2 – Wiring between Platform Abstraction and Implementation</strong></p>
-<p>The second layer builds the bridge between the WayangContext and PlanBuilders which work together with DataQuanta and the DataQuantaBuilder.</p>
-<p>Also, the mapping between the abstract components and the specific implementations are defined in this layer.</p>
-<p>Therefore, the mappings package has a class <em>Mappings</em> in which all relevant input and output operators are listed.
-We use it to register the KafkaSourceMapping and a KafkaSinkMapping for the particular platform, Java in our case.
-These classes allow the Apache Wayang framework to use the Java implementation of the KafkaTopicSource component (and KafkaTopicSink respectively).
-While the Wayang execution plan uses the higher abstractions, here on the “platform level” we have to link the specific implementation for the target platform.
-In our case this leads to a Java program running on a JVM which is set up by the Apache Wayang framework using the logical components of the execution plan.</p>
-<p>Those mappings link the real implementation of our operators the ones used in an execution plan.
-The JavaKafkaTopicSource and the JavaKafkaTopicSink extend the KafkaTopicSource and KafkaTopicSink so that the lower level implementation of those classes become available within Wayang’s Java Platform context.</p>
-<p>In this layer, the KafkaConsumer class and the KafkaProducer class are used, but both are configured and instantiated in the next layer underneath.
-All this is done in the project <em>wayang-plarforms/wayang-java</em>.</p>
-<p><strong>Layer 3 – Input/Output Connector Layer</strong></p>
-<p>The <em>KafkaTopicSource</em> and <em>KafkaTopicSink</em> classes build the third layer of our implementation.
-Both are implemented in Java programming language.
-In this layer, the real Kafka-Client logic is defined.
-Details about consumer and producers, client configuration, and schema handling have to be handled here.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="summary">Summary<a href="#summary" class="hash-link" aria-label="Direct link to Summary" title="Direct link to Summary"></a></h2>
-<p>Both classes in the third layer implement the Kafka client logic which is needed by the Wayang-execution plan when external data flows should be established.
-The layer above handles the mapping of the components at startup time.
-All this wiring is needed to keep Wayang open and flexible so that multiple external systems can be used in a variety of combinations and using multiple target platforms in combinations.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="outlook">Outlook<a href="#outlook" class="hash-link" aria-label="Direct link to Outlook" title="Direct link to Outlook"></a></h2>
-<p>The next part of the article series will cover the creation of an Kafka Source and Sink component for the Apache Spark platform, which allows our use case to scale.
-Finally, in part four we bring all puzzles together, and show the full implementation of the multi organizational data collaboration use case.</p></div><footer class="row docusaurus-mt-lg"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/kafka">kafka</a></li><li class="tag_QGVx"><a class= [...]
-We look into the “Read- and Write-Path” for our data items, called <em>DataQuanta</em>.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="apache-wayangs-read--write-path-for-kafka-topics">Apache Wayang’s Read & Write Path for Kafka topics<a href="#apache-wayangs-read--write-path-for-kafka-topics" class="hash-link" aria-label="Direct link to Apache Wayang’s Read & Write Path for Kafka topics" title="Direct link to Apache Wayang’s Read & Write Path for Kafka topics"></a></h2>
-<p>To describe the read and write paths for data in the context of the created Apache Wayang code snippet, the primary classes and interfaces we need to understand are as follows:</p>
-<p><strong>WayangContext:</strong> This class is essential for initializing the Wayang processing environment.
-It allows you to configure the execution environment and register plugins that define which platforms Wayang can use for data processing tasks, such as <em>Java.basicPlugin()</em> for local Java execution.</p>
-<p><strong>JavaPlanBuilder:</strong> This class is used to build and define the data processing pipeline (or plan) in Wayang.
-It provides a fluent API to specify the operations to be performed on the data, from reading the input to processing it and writing the output.</p>
-<h3 class="anchor anchorWithStickyNavbar_LWe7" id="read-path">Read Path<a href="#read-path" class="hash-link" aria-label="Direct link to Read Path" title="Direct link to Read Path"></a></h3>
-<p>The read path describes how data is ingested from a source into the Wayang processing pipeline:</p>
-<p><em>Reading from Kafka Topic:</em> The method <em>readKafkaTopic(topicName)</em> is used to ingest data from a specified Kafka topic.
-This is the starting point of the data processing pipeline, where topicName represents the name of the Kafka topic from which data is read.</p>
-<p><em>Data Tokenization and Preparation:</em> Once the data is read from Kafka, it undergoes several transformations such as Splitting, Filtering, and Mapping.
-What follows are the procedures known as Reducing, Grouping, Co-Grouping, and Counting.</p>
-<h3 class="anchor anchorWithStickyNavbar_LWe7" id="write-path">Write Path<a href="#write-path" class="hash-link" aria-label="Direct link to Write Path" title="Direct link to Write Path"></a></h3>
-<p><em>Writing to Kafka Topic:</em> The final step in the pipeline involves writing the processed data back to a Kafka topic using <em>.writeKafkaTopic(...)</em>.
-This method takes parameters that specify the target Kafka topic, a serialization function to format the data as strings, and additional configuration for load profile estimation, which optimizes the writing process.</p>
-<p>This read-write path provides a comprehensive flow of data from ingestion from Kafka, through various processing steps, and finally back to Kafka, showcasing a full cycle of data processing within Apache Wayang's abstracted environment and is implemented in our example program shown in <em>listing 1</em>.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="implementation-of-input--and-output-operators">Implementation of Input- and Output Operators<a href="#implementation-of-input--and-output-operators" class="hash-link" aria-label="Direct link to Implementation of Input- and Output Operators" title="Direct link to Implementation of Input- and Output Operators"></a></h2>
-<p>The next section shows how a new pair of operators can be implemented to extend Apache Wayang’s capabilities on the input and output side.
-We created the Kafka Source and Kafka Sink components so that our cross organizational data collaboration scenario can be implemented using data streaming infrastructure.</p>
-<p><strong>Level 1 – Wayang execution plan with abstract operators</strong></p>
-<p>The implementation of our Kafka Source and Kafka Sink components for Apache Wayang requires new methods and classes on three layers.
-First of all in the API package.
-Here we use the JavaPlanBuilder to expose the function for selecting a Kafka topic as the source to be used by client.
-The class <em>JavaPlanBuilder</em> in package <em>org.apache.wayang.api</em> in the project <em>wayang-api/wayang-api-scala-java</em> exposes our new functionality to our external client.
-An instance of the JavaPlanBuilder is used to define the data processing pipeline.
-We use its <em>readKafkaTopic()</em> which specifies the source Kafka topic to read from, and for the write path we use the <em>writeKafkaTopic()</em> method.
-Both Methods do only trigger activities in the background.</p>
-<p>For the output side, we use the <em>DataQuantaBuilder</em> class, which offers an implementation of the writeKafkaTopic function.
-This function is designed to send processed data, referred to as DataQuanta, to a specified Kafka topic.
-Essentially, it marks the final step in a data processing sequence constructed using the Apache Wayang framework.</p>
-<p>In the DataQuanta class we implemented the methods writeKafkaTopic and writeKafkaTopicJava which use the KafkaTopicSink class.
-In this API layer we use the Scala programming language, but we utilize the Java classes, implemented in the layer below.</p>
-<p><strong>Level 2 – Wiring between Platform Abstraction and Implementation</strong></p>
-<p>The second layer builds the bridge between the WayangContext and PlanBuilders which work together with DataQuanta and the DataQuantaBuilder.</p>
-<p>Also, the mapping between the abstract components and the specific implementations are defined in this layer.</p>
-<p>Therefore, the mappings package has a class <em>Mappings</em> in which all relevant input and output operators are listed.
-We use it to register the KafkaSourceMapping and a KafkaSinkMapping for the particular platform, Java in our case.
-These classes allow the Apache Wayang framework to use the Java implementation of the KafkaTopicSource component (and KafkaTopicSink respectively).
-While the Wayang execution plan uses the higher abstractions, here on the “platform level” we have to link the specific implementation for the target platform.
-In our case this leads to a Java program running on a JVM which is set up by the Apache Wayang framework using the logical components of the execution plan.</p>
-<p>Those mappings link the real implementation of our operators the ones used in an execution plan.
-The JavaKafkaTopicSource and the JavaKafkaTopicSink extend the KafkaTopicSource and KafkaTopicSink so that the lower level implementation of those classes become available within Wayang’s Java Platform context.</p>
-<p>In this layer, the KafkaConsumer class and the KafkaProducer class are used, but both are configured and instantiated in the next layer underneath.
-All this is done in the project <em>wayang-plarforms/wayang-java</em>.</p>
-<p><strong>Layer 3 – Input/Output Connector Layer</strong></p>
-<p>The <em>KafkaTopicSource</em> and <em>KafkaTopicSink</em> classes build the third layer of our implementation.
-Both are implemented in Java programming language.
-In this layer, the real Kafka-Client logic is defined.
-Details about consumer and producers, client configuration, and schema handling have to be handled here.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="summary">Summary<a href="#summary" class="hash-link" aria-label="Direct link to Summary" title="Direct link to Summary"></a></h2>
-<p>Both classes in the third layer implement the Kafka client logic which is needed by the Wayang-execution plan when external data flows should be established.
-The layer above handles the mapping of the components at startup time.
-All this wiring is needed to keep Wayang open and flexible so that multiple external systems can be used in a variety of combinations and using multiple target platforms in combinations.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="outlook">Outlook<a href="#outlook" class="hash-link" aria-label="Direct link to Outlook" title="Direct link to Outlook"></a></h2>
-<p>The next part of the article series will cover the creation of an Kafka Source and Sink component for the Apache Spark platform, which allows our use case to scale.
-Finally, in part four we bring all puzzles together, and show the full implementation of the multi organizational data collaboration use case.</p></div><footer class="row docusaurus-mt-lg"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/kafka">kafka</a></li><li class="tag_QGVx"><a class= [...]
+<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
Motivated by the learnings from last time, I stated implementing a Kafka Source component and a Kafka Sink component for the Apache Spark platform in Apache Wayang.
In our previous article we shared the results of the work on the frist Apache Kafka integration using the Java Platform.</p>
<p>Let's see how it goes this time with Apache Spark.</p>
@@ -234,7 +77,7 @@ More testing, more serialization schemes, and Kafka Schema Registry support shou
<h2 class="anchor anchorWithStickyNavbar_LWe7" id="outlook">Outlook<a href="#outlook" class="hash-link" aria-label="Direct link to Outlook" title="Direct link to Outlook"></a></h2>
<p>The next part of the article series will cover the real world example as described in image 1.
We will show how analysts and developers can use the Apache Kafka integration for Apache Wayang to solve cross organizational collaboration issues.
-Therefore, we will bring all puzzles together, and show the full implementation of the multi organizational data collaboration use case.</p></div><footer class="row docusaurus-mt-lg"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/kafka">kafka</a></li><li class="tag_QGVx"><a class="tag_z [...]
+Therefore, we will bring all puzzles together, and show the full implementation of the multi organizational data collaboration use case.</p></div><footer class="row docusaurus-mt-lg"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/kafka">kafka</a></li><li class="tag_QGVx"><a class="tag_z [...]
We look into the “Read- and Write-Path” for our data items, called <em>DataQuanta</em>.</p>
<h2 class="anchor anchorWithStickyNavbar_LWe7" id="apache-wayangs-read--write-path-for-kafka-topics">Apache Wayang’s Read & Write Path for Kafka topics<a href="#apache-wayangs-read--write-path-for-kafka-topics" class="hash-link" aria-label="Direct link to Apache Wayang’s Read & Write Path for Kafka topics" title="Direct link to Apache Wayang’s Read & Write Path for Kafka topics"></a></h2>
<p>To describe the read and write paths for data in the context of the created Apache Wayang code snippet, the primary classes and interfaces we need to understand are as follows:</p>
@@ -291,7 +134,7 @@ The layer above handles the mapping of the components at startup time.
All this wiring is needed to keep Wayang open and flexible so that multiple external systems can be used in a variety of combinations and using multiple target platforms in combinations.</p>
<h2 class="anchor anchorWithStickyNavbar_LWe7" id="outlook">Outlook<a href="#outlook" class="hash-link" aria-label="Direct link to Outlook" title="Direct link to Outlook"></a></h2>
<p>The next part of the article series will cover the creation of an Kafka Source and Sink component for the Apache Spark platform, which allows our use case to scale.
-Finally, in part four we bring all puzzles together, and show the full implementation of the multi organizational data collaboration use case.</p></div><footer class="row docusaurus-mt-lg"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/kafka">kafka</a></li><li class="tag_QGVx"><a class= [...]
+Finally, in part four we bring all puzzles together, and show the full implementation of the multi organizational data collaboration use case.</p></div><footer class="row docusaurus-mt-lg"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/kafka">kafka</a></li><li class="tag_QGVx"><a class= [...]
<p>This article is the first of a four part series about federated data analysis using Apache Wayang.
The first article starts with an introduction of a typical data colaboration scenario which will emerge in our digital future.</p>
<p>In part two and three we will share a summary of our Apache Kafka client implementation for Apache Wayang.
diff --git a/blog/tags/federated/index.html b/blog/tags/federated/index.html
index 73e95159..be41865f 100644
--- a/blog/tags/federated/index.html
+++ b/blog/tags/federated/index.html
@@ -5,11 +5,11 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">One post tagged with "federated" | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/blog/tags/federated"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="do [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.01bb8940.js" defer="defer"></script>
-<script src="/assets/js/main.0cc2e93a.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.8489d39d.js" defer="defer"></script>
+<script src="/assets/js/main.4f5176d9.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
-<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
+<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
detecting early signs of cancer cells. Building an AI requires a lot of data and training to achieve
the desired results, and federated learning is an approach to make AI training more viable.
Federated learning (or collaborative learning) is a technique that trains AI models on data
diff --git a/blog/tags/index.html b/blog/tags/index.html
index 8fbd06a6..68865209 100644
--- a/blog/tags/index.html
+++ b/blog/tags/index.html
@@ -5,11 +5,11 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">Tags | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/blog/tags"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language" content="en"><meta data-rh="tr [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.01bb8940.js" defer="defer"></script>
-<script src="/assets/js/main.0cc2e93a.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.8489d39d.js" defer="defer"></script>
+<script src="/assets/js/main.4f5176d9.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
-<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
+<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
<p> Apache Wayang is an effort undergoing incubation at The Apache Software Foundation (ASF), sponsored by the Apache Incubator. Incubation is required of all newly accepted projects until a further review indicates that the infrastructure, communications, and decision making process have stabilized in a manner consistent with other successful ASF projects. While incubation status is not necessarily a reflection of the completeness or stability of the code, it does indicate that th [...]
<p>
Copyright © 2024 The Apache Software Foundation, Licensed under the Apache License, Version 2.0. <br>
diff --git a/blog/tags/kafka/index.html b/blog/tags/kafka/index.html
index 231ea394..0c0d00f4 100644
--- a/blog/tags/kafka/index.html
+++ b/blog/tags/kafka/index.html
@@ -3,170 +3,13 @@
<head>
<meta charset="UTF-8">
<meta name="generator" content="Docusaurus v3.1.1">
-<title data-rh="true">6 posts tagged with "kafka" | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/blog/tags/kafka"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:l [...]
+<title data-rh="true">3 posts tagged with "kafka" | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/blog/tags/kafka"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:l [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.01bb8940.js" defer="defer"></script>
-<script src="/assets/js/main.0cc2e93a.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.8489d39d.js" defer="defer"></script>
+<script src="/assets/js/main.4f5176d9.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
-<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
-<p>This article is the first of a four part series about federated data analysis using Apache Wayang.
-The first article starts with an introduction of a typical data colaboration scenario which will emerge in our digital future.</p>
-<p>In part two and three we will share a summary of our Apache Kafka client implementation for Apache Wayang.
-We started with the Java Platform (part 2) and the Apache Spark implementation follows (W.I.P.) in part three.</p>
-<p>The use case behind this work is an imaginary data collaboration scenario.
-We see this example and the demand for a solution already in many places.
-For us this is motivation enough to propose a solution.
-This would also allow us to do more local data processing, and businesses can stop moving data around the world, but rather care about data locality while they expose and share specific information to others by using data federation.
-This reduces complexity of data management and cost dramatically.</p>
-<p>For this purpose, we illustrate a cross organizational data sharing scenario from the finance sector soon.
-This analysis pattern will also be relevant in the context of data analysis along supply chains, another typical example where data from many stakeholder together is needed but never managed in one place, for good reasons.</p>
-<p>Data federation can help us to unlock the hidden value of all those isolated data lakes.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="a-cross-organizational-data-sharing-scenario">A cross organizational data sharing scenario<a href="#a-cross-organizational-data-sharing-scenario" class="hash-link" aria-label="Direct link to A cross organizational data sharing scenario" title="Direct link to A cross organizational data sharing scenario"></a></h2>
-<p>Our goal is the implementation of a cross organization decentralized data processing scenario, in which protected local data should be processed in combination with public data from public sources in a collaborative manner.
-Instead of copying all data into a central data lake or a central data platform we decided to use federated analytics.
-Apache Wayang is the tool we work with.
-In our case, the public data is hosted on publicly available websites or data pods.
-A client can use the HTTP(S) protocol to read the data which is given in a well defined format.
-For simplicity we decided to use CSV format.
-When we look into the data of each participant we have a different perspective.</p>
-<p>Our processing procedure should calculate a particular metric on the <em>local data</em> of each participant.
-An example of such a metric is the average spending of all users on a particular product category per month.
-This can vary from partner to partner, hence, we want to be able to calculate a peer-group comparison so that each partner can see its own metric compared with a global average calculated from contributions by all partners.
-Such a process requires global averaging and local averaging.
-And due to governance constraints, we can’t bring all raw data together in one place.</p>
-<p>Instead, we want to use Apache Wayang for this purpose.
-We simplify the procedure and split it into two phases.
-Phase one is the process, which allows each participant to calculate the local metrics.
-This requires only local data. The second phase requires data from all collaborating partners.
-The monthly sum and counter values per partner and category are needed in one place by all other parties.
-Hence, the algorithm of the first phase stores the local results locally, and the contributions to the global results in an externally accessible Kafka topic.
-We assume this is done by each of the partners.</p>
-<p>Now we have a scenario, in which an Apache Wayang process must be able to read data from multiple Apache Kafka topics from multiple Apache Kafka clusters but finally writes into a single Kafka topic, which then can be accessed by all the participating clients.</p>
-<p><img decoding="async" loading="lazy" alt="images/image-1.png" src="/assets/images/image-1-9cc35d5aea2b867d7e5759a96bd02334.png" width="904" height="550" class="img_ev3q"></p>
-<p>The illustration shows the data flows in such a scenario.
-Jobs with red border are executed by the participants in isolation within their own data processing environments.
-But they share some of the data, using publicly accessible Kafka topics, marked by A. Job 4 is the Apache Wayang job in our focus: here we intent to read data from 3 different source systems, and write results into a fourth system (marked as B), which can be accesses by all participants again.</p>
-<p>With this in mind we want to implement an Apache Wayang application which implements the illustrated <em>Job 4</em>.
-Since as of today, there is now <em>KafkaSource</em> and <em>KafkaSink</em> available in Apache Wayang, an implementation of both will be our first step.
-Our assumption is, that in the beginning, there won’t be much data.</p>
-<p>Apache Spark is not required to cope with the load, but we expect, that in the future, a single Java application would not be able to handle our workload.
-Hence, we want to utilize the Apache Wayang abstraction over multiple processing platforms, starting with Java.
-Later, we want to switch to Apache Spark.</p></div><footer class="row docusaurus-mt-lg"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/kafka">kafka</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/cross-organization-data-collaboration">cross organization [...]
-We look into the “Read- and Write-Path” for our data items, called <em>DataQuanta</em>.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="apache-wayangs-read--write-path-for-kafka-topics">Apache Wayang’s Read & Write Path for Kafka topics<a href="#apache-wayangs-read--write-path-for-kafka-topics" class="hash-link" aria-label="Direct link to Apache Wayang’s Read & Write Path for Kafka topics" title="Direct link to Apache Wayang’s Read & Write Path for Kafka topics"></a></h2>
-<p>To describe the read and write paths for data in the context of the created Apache Wayang code snippet, the primary classes and interfaces we need to understand are as follows:</p>
-<p><strong>WayangContext:</strong> This class is essential for initializing the Wayang processing environment.
-It allows you to configure the execution environment and register plugins that define which platforms Wayang can use for data processing tasks, such as <em>Java.basicPlugin()</em> for local Java execution.</p>
-<p><strong>JavaPlanBuilder:</strong> This class is used to build and define the data processing pipeline (or plan) in Wayang.
-It provides a fluent API to specify the operations to be performed on the data, from reading the input to processing it and writing the output.</p>
-<h3 class="anchor anchorWithStickyNavbar_LWe7" id="read-path">Read Path<a href="#read-path" class="hash-link" aria-label="Direct link to Read Path" title="Direct link to Read Path"></a></h3>
-<p>The read path describes how data is ingested from a source into the Wayang processing pipeline:</p>
-<p><em>Reading from Kafka Topic:</em> The method <em>readKafkaTopic(topicName)</em> is used to ingest data from a specified Kafka topic.
-This is the starting point of the data processing pipeline, where topicName represents the name of the Kafka topic from which data is read.</p>
-<p><em>Data Tokenization and Preparation:</em> Once the data is read from Kafka, it undergoes several transformations such as Splitting, Filtering, and Mapping.
-What follows are the procedures known as Reducing, Grouping, Co-Grouping, and Counting.</p>
-<h3 class="anchor anchorWithStickyNavbar_LWe7" id="write-path">Write Path<a href="#write-path" class="hash-link" aria-label="Direct link to Write Path" title="Direct link to Write Path"></a></h3>
-<p><em>Writing to Kafka Topic:</em> The final step in the pipeline involves writing the processed data back to a Kafka topic using <em>.writeKafkaTopic(...)</em>.
-This method takes parameters that specify the target Kafka topic, a serialization function to format the data as strings, and additional configuration for load profile estimation, which optimizes the writing process.</p>
-<p>This read-write path provides a comprehensive flow of data from ingestion from Kafka, through various processing steps, and finally back to Kafka, showcasing a full cycle of data processing within Apache Wayang's abstracted environment and is implemented in our example program shown in <em>listing 1</em>.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="implementation-of-input--and-output-operators">Implementation of Input- and Output Operators<a href="#implementation-of-input--and-output-operators" class="hash-link" aria-label="Direct link to Implementation of Input- and Output Operators" title="Direct link to Implementation of Input- and Output Operators"></a></h2>
-<p>The next section shows how a new pair of operators can be implemented to extend Apache Wayang’s capabilities on the input and output side.
-We created the Kafka Source and Kafka Sink components so that our cross organizational data collaboration scenario can be implemented using data streaming infrastructure.</p>
-<p><strong>Level 1 – Wayang execution plan with abstract operators</strong></p>
-<p>The implementation of our Kafka Source and Kafka Sink components for Apache Wayang requires new methods and classes on three layers.
-First of all in the API package.
-Here we use the JavaPlanBuilder to expose the function for selecting a Kafka topic as the source to be used by client.
-The class <em>JavaPlanBuilder</em> in package <em>org.apache.wayang.api</em> in the project <em>wayang-api/wayang-api-scala-java</em> exposes our new functionality to our external client.
-An instance of the JavaPlanBuilder is used to define the data processing pipeline.
-We use its <em>readKafkaTopic()</em> which specifies the source Kafka topic to read from, and for the write path we use the <em>writeKafkaTopic()</em> method.
-Both Methods do only trigger activities in the background.</p>
-<p>For the output side, we use the <em>DataQuantaBuilder</em> class, which offers an implementation of the writeKafkaTopic function.
-This function is designed to send processed data, referred to as DataQuanta, to a specified Kafka topic.
-Essentially, it marks the final step in a data processing sequence constructed using the Apache Wayang framework.</p>
-<p>In the DataQuanta class we implemented the methods writeKafkaTopic and writeKafkaTopicJava which use the KafkaTopicSink class.
-In this API layer we use the Scala programming language, but we utilize the Java classes, implemented in the layer below.</p>
-<p><strong>Level 2 – Wiring between Platform Abstraction and Implementation</strong></p>
-<p>The second layer builds the bridge between the WayangContext and PlanBuilders which work together with DataQuanta and the DataQuantaBuilder.</p>
-<p>Also, the mapping between the abstract components and the specific implementations are defined in this layer.</p>
-<p>Therefore, the mappings package has a class <em>Mappings</em> in which all relevant input and output operators are listed.
-We use it to register the KafkaSourceMapping and a KafkaSinkMapping for the particular platform, Java in our case.
-These classes allow the Apache Wayang framework to use the Java implementation of the KafkaTopicSource component (and KafkaTopicSink respectively).
-While the Wayang execution plan uses the higher abstractions, here on the “platform level” we have to link the specific implementation for the target platform.
-In our case this leads to a Java program running on a JVM which is set up by the Apache Wayang framework using the logical components of the execution plan.</p>
-<p>Those mappings link the real implementation of our operators the ones used in an execution plan.
-The JavaKafkaTopicSource and the JavaKafkaTopicSink extend the KafkaTopicSource and KafkaTopicSink so that the lower level implementation of those classes become available within Wayang’s Java Platform context.</p>
-<p>In this layer, the KafkaConsumer class and the KafkaProducer class are used, but both are configured and instantiated in the next layer underneath.
-All this is done in the project <em>wayang-plarforms/wayang-java</em>.</p>
-<p><strong>Layer 3 – Input/Output Connector Layer</strong></p>
-<p>The <em>KafkaTopicSource</em> and <em>KafkaTopicSink</em> classes build the third layer of our implementation.
-Both are implemented in Java programming language.
-In this layer, the real Kafka-Client logic is defined.
-Details about consumer and producers, client configuration, and schema handling have to be handled here.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="summary">Summary<a href="#summary" class="hash-link" aria-label="Direct link to Summary" title="Direct link to Summary"></a></h2>
-<p>Both classes in the third layer implement the Kafka client logic which is needed by the Wayang-execution plan when external data flows should be established.
-The layer above handles the mapping of the components at startup time.
-All this wiring is needed to keep Wayang open and flexible so that multiple external systems can be used in a variety of combinations and using multiple target platforms in combinations.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="outlook">Outlook<a href="#outlook" class="hash-link" aria-label="Direct link to Outlook" title="Direct link to Outlook"></a></h2>
-<p>The next part of the article series will cover the creation of an Kafka Source and Sink component for the Apache Spark platform, which allows our use case to scale.
-Finally, in part four we bring all puzzles together, and show the full implementation of the multi organizational data collaboration use case.</p></div><footer class="row docusaurus-mt-lg"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/kafka">kafka</a></li><li class="tag_QGVx"><a class= [...]
-We look into the “Read- and Write-Path” for our data items, called <em>DataQuanta</em>.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="apache-wayangs-read--write-path-for-kafka-topics">Apache Wayang’s Read & Write Path for Kafka topics<a href="#apache-wayangs-read--write-path-for-kafka-topics" class="hash-link" aria-label="Direct link to Apache Wayang’s Read & Write Path for Kafka topics" title="Direct link to Apache Wayang’s Read & Write Path for Kafka topics"></a></h2>
-<p>To describe the read and write paths for data in the context of the created Apache Wayang code snippet, the primary classes and interfaces we need to understand are as follows:</p>
-<p><strong>WayangContext:</strong> This class is essential for initializing the Wayang processing environment.
-It allows you to configure the execution environment and register plugins that define which platforms Wayang can use for data processing tasks, such as <em>Java.basicPlugin()</em> for local Java execution.</p>
-<p><strong>JavaPlanBuilder:</strong> This class is used to build and define the data processing pipeline (or plan) in Wayang.
-It provides a fluent API to specify the operations to be performed on the data, from reading the input to processing it and writing the output.</p>
-<h3 class="anchor anchorWithStickyNavbar_LWe7" id="read-path">Read Path<a href="#read-path" class="hash-link" aria-label="Direct link to Read Path" title="Direct link to Read Path"></a></h3>
-<p>The read path describes how data is ingested from a source into the Wayang processing pipeline:</p>
-<p><em>Reading from Kafka Topic:</em> The method <em>readKafkaTopic(topicName)</em> is used to ingest data from a specified Kafka topic.
-This is the starting point of the data processing pipeline, where topicName represents the name of the Kafka topic from which data is read.</p>
-<p><em>Data Tokenization and Preparation:</em> Once the data is read from Kafka, it undergoes several transformations such as Splitting, Filtering, and Mapping.
-What follows are the procedures known as Reducing, Grouping, Co-Grouping, and Counting.</p>
-<h3 class="anchor anchorWithStickyNavbar_LWe7" id="write-path">Write Path<a href="#write-path" class="hash-link" aria-label="Direct link to Write Path" title="Direct link to Write Path"></a></h3>
-<p><em>Writing to Kafka Topic:</em> The final step in the pipeline involves writing the processed data back to a Kafka topic using <em>.writeKafkaTopic(...)</em>.
-This method takes parameters that specify the target Kafka topic, a serialization function to format the data as strings, and additional configuration for load profile estimation, which optimizes the writing process.</p>
-<p>This read-write path provides a comprehensive flow of data from ingestion from Kafka, through various processing steps, and finally back to Kafka, showcasing a full cycle of data processing within Apache Wayang's abstracted environment and is implemented in our example program shown in <em>listing 1</em>.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="implementation-of-input--and-output-operators">Implementation of Input- and Output Operators<a href="#implementation-of-input--and-output-operators" class="hash-link" aria-label="Direct link to Implementation of Input- and Output Operators" title="Direct link to Implementation of Input- and Output Operators"></a></h2>
-<p>The next section shows how a new pair of operators can be implemented to extend Apache Wayang’s capabilities on the input and output side.
-We created the Kafka Source and Kafka Sink components so that our cross organizational data collaboration scenario can be implemented using data streaming infrastructure.</p>
-<p><strong>Level 1 – Wayang execution plan with abstract operators</strong></p>
-<p>The implementation of our Kafka Source and Kafka Sink components for Apache Wayang requires new methods and classes on three layers.
-First of all in the API package.
-Here we use the JavaPlanBuilder to expose the function for selecting a Kafka topic as the source to be used by client.
-The class <em>JavaPlanBuilder</em> in package <em>org.apache.wayang.api</em> in the project <em>wayang-api/wayang-api-scala-java</em> exposes our new functionality to our external client.
-An instance of the JavaPlanBuilder is used to define the data processing pipeline.
-We use its <em>readKafkaTopic()</em> which specifies the source Kafka topic to read from, and for the write path we use the <em>writeKafkaTopic()</em> method.
-Both Methods do only trigger activities in the background.</p>
-<p>For the output side, we use the <em>DataQuantaBuilder</em> class, which offers an implementation of the writeKafkaTopic function.
-This function is designed to send processed data, referred to as DataQuanta, to a specified Kafka topic.
-Essentially, it marks the final step in a data processing sequence constructed using the Apache Wayang framework.</p>
-<p>In the DataQuanta class we implemented the methods writeKafkaTopic and writeKafkaTopicJava which use the KafkaTopicSink class.
-In this API layer we use the Scala programming language, but we utilize the Java classes, implemented in the layer below.</p>
-<p><strong>Level 2 – Wiring between Platform Abstraction and Implementation</strong></p>
-<p>The second layer builds the bridge between the WayangContext and PlanBuilders which work together with DataQuanta and the DataQuantaBuilder.</p>
-<p>Also, the mapping between the abstract components and the specific implementations are defined in this layer.</p>
-<p>Therefore, the mappings package has a class <em>Mappings</em> in which all relevant input and output operators are listed.
-We use it to register the KafkaSourceMapping and a KafkaSinkMapping for the particular platform, Java in our case.
-These classes allow the Apache Wayang framework to use the Java implementation of the KafkaTopicSource component (and KafkaTopicSink respectively).
-While the Wayang execution plan uses the higher abstractions, here on the “platform level” we have to link the specific implementation for the target platform.
-In our case this leads to a Java program running on a JVM which is set up by the Apache Wayang framework using the logical components of the execution plan.</p>
-<p>Those mappings link the real implementation of our operators the ones used in an execution plan.
-The JavaKafkaTopicSource and the JavaKafkaTopicSink extend the KafkaTopicSource and KafkaTopicSink so that the lower level implementation of those classes become available within Wayang’s Java Platform context.</p>
-<p>In this layer, the KafkaConsumer class and the KafkaProducer class are used, but both are configured and instantiated in the next layer underneath.
-All this is done in the project <em>wayang-plarforms/wayang-java</em>.</p>
-<p><strong>Layer 3 – Input/Output Connector Layer</strong></p>
-<p>The <em>KafkaTopicSource</em> and <em>KafkaTopicSink</em> classes build the third layer of our implementation.
-Both are implemented in Java programming language.
-In this layer, the real Kafka-Client logic is defined.
-Details about consumer and producers, client configuration, and schema handling have to be handled here.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="summary">Summary<a href="#summary" class="hash-link" aria-label="Direct link to Summary" title="Direct link to Summary"></a></h2>
-<p>Both classes in the third layer implement the Kafka client logic which is needed by the Wayang-execution plan when external data flows should be established.
-The layer above handles the mapping of the components at startup time.
-All this wiring is needed to keep Wayang open and flexible so that multiple external systems can be used in a variety of combinations and using multiple target platforms in combinations.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="outlook">Outlook<a href="#outlook" class="hash-link" aria-label="Direct link to Outlook" title="Direct link to Outlook"></a></h2>
-<p>The next part of the article series will cover the creation of an Kafka Source and Sink component for the Apache Spark platform, which allows our use case to scale.
-Finally, in part four we bring all puzzles together, and show the full implementation of the multi organizational data collaboration use case.</p></div><footer class="row docusaurus-mt-lg"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/kafka">kafka</a></li><li class="tag_QGVx"><a class= [...]
+<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
Motivated by the learnings from last time, I stated implementing a Kafka Source component and a Kafka Sink component for the Apache Spark platform in Apache Wayang.
In our previous article we shared the results of the work on the frist Apache Kafka integration using the Java Platform.</p>
<p>Let's see how it goes this time with Apache Spark.</p>
@@ -234,7 +77,7 @@ More testing, more serialization schemes, and Kafka Schema Registry support shou
<h2 class="anchor anchorWithStickyNavbar_LWe7" id="outlook">Outlook<a href="#outlook" class="hash-link" aria-label="Direct link to Outlook" title="Direct link to Outlook"></a></h2>
<p>The next part of the article series will cover the real world example as described in image 1.
We will show how analysts and developers can use the Apache Kafka integration for Apache Wayang to solve cross organizational collaboration issues.
-Therefore, we will bring all puzzles together, and show the full implementation of the multi organizational data collaboration use case.</p></div><footer class="row docusaurus-mt-lg"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/kafka">kafka</a></li><li class="tag_QGVx"><a class="tag_z [...]
+Therefore, we will bring all puzzles together, and show the full implementation of the multi organizational data collaboration use case.</p></div><footer class="row docusaurus-mt-lg"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/kafka">kafka</a></li><li class="tag_QGVx"><a class="tag_z [...]
We look into the “Read- and Write-Path” for our data items, called <em>DataQuanta</em>.</p>
<h2 class="anchor anchorWithStickyNavbar_LWe7" id="apache-wayangs-read--write-path-for-kafka-topics">Apache Wayang’s Read & Write Path for Kafka topics<a href="#apache-wayangs-read--write-path-for-kafka-topics" class="hash-link" aria-label="Direct link to Apache Wayang’s Read & Write Path for Kafka topics" title="Direct link to Apache Wayang’s Read & Write Path for Kafka topics"></a></h2>
<p>To describe the read and write paths for data in the context of the created Apache Wayang code snippet, the primary classes and interfaces we need to understand are as follows:</p>
@@ -291,7 +134,7 @@ The layer above handles the mapping of the components at startup time.
All this wiring is needed to keep Wayang open and flexible so that multiple external systems can be used in a variety of combinations and using multiple target platforms in combinations.</p>
<h2 class="anchor anchorWithStickyNavbar_LWe7" id="outlook">Outlook<a href="#outlook" class="hash-link" aria-label="Direct link to Outlook" title="Direct link to Outlook"></a></h2>
<p>The next part of the article series will cover the creation of an Kafka Source and Sink component for the Apache Spark platform, which allows our use case to scale.
-Finally, in part four we bring all puzzles together, and show the full implementation of the multi organizational data collaboration use case.</p></div><footer class="row docusaurus-mt-lg"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/kafka">kafka</a></li><li class="tag_QGVx"><a class= [...]
+Finally, in part four we bring all puzzles together, and show the full implementation of the multi organizational data collaboration use case.</p></div><footer class="row docusaurus-mt-lg"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/kafka">kafka</a></li><li class="tag_QGVx"><a class= [...]
<p>This article is the first of a four part series about federated data analysis using Apache Wayang.
The first article starts with an introduction of a typical data colaboration scenario which will emerge in our digital future.</p>
<p>In part two and three we will share a summary of our Apache Kafka client implementation for Apache Wayang.
diff --git a/blog/tags/ml/index.html b/blog/tags/ml/index.html
index a4d43440..6adb220c 100644
--- a/blog/tags/ml/index.html
+++ b/blog/tags/ml/index.html
@@ -5,11 +5,11 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">One post tagged with "ML" | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/blog/tags/ml"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:langua [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.01bb8940.js" defer="defer"></script>
-<script src="/assets/js/main.0cc2e93a.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.8489d39d.js" defer="defer"></script>
+<script src="/assets/js/main.4f5176d9.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
-<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
+<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
Thanks to the extensible nature of Wayang, the only core changes we had to do were introducing the concept of a <code>Model</code> and implement a new driver for the newly added platform.</p>
<h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-1-introducing-a-model">Step 1: Introducing a Model<a href="#step-1-introducing-a-model" class="hash-link" aria-label="Direct link to Step 1: Introducing a Model" title="Direct link to Step 1: Introducing a Model"></a></h2>
<p>With respect to the model, we followed Wayang’s abstraction philosophy: We created a <code>Model</code> interface to be used as input or output by Wayang operators and then extended it for the platform-specific operators. Different model interfaces can be found here:</p>
diff --git a/blog/tags/presto/index.html b/blog/tags/presto/index.html
index 7643dcbe..1c2be9e9 100644
--- a/blog/tags/presto/index.html
+++ b/blog/tags/presto/index.html
@@ -5,11 +5,11 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">One post tagged with "presto" | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/blog/tags/presto"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearc [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.01bb8940.js" defer="defer"></script>
-<script src="/assets/js/main.0cc2e93a.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.8489d39d.js" defer="defer"></script>
+<script src="/assets/js/main.4f5176d9.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
-<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
+<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
<p> Apache Wayang is an effort undergoing incubation at The Apache Software Foundation (ASF), sponsored by the Apache Incubator. Incubation is required of all newly accepted projects until a further review indicates that the infrastructure, communications, and decision making process have stabilized in a manner consistent with other successful ASF projects. While incubation status is not necessarily a reflection of the completeness or stability of the code, it does indicate that th [...]
<p>
Copyright © 2024 The Apache Software Foundation, Licensed under the Apache License, Version 2.0. <br>
diff --git a/blog/tags/python/index.html b/blog/tags/python/index.html
index 57fc4a67..6486046c 100644
--- a/blog/tags/python/index.html
+++ b/blog/tags/python/index.html
@@ -5,11 +5,11 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">One post tagged with "python" | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/blog/tags/python"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearc [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.01bb8940.js" defer="defer"></script>
-<script src="/assets/js/main.0cc2e93a.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.8489d39d.js" defer="defer"></script>
+<script src="/assets/js/main.4f5176d9.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
-<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
+<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
important. However, navigating through a multitude of tools and
languages often is a major inconvenience.
Apache Wayang's upcoming Python API will allow you to seamlessly
diff --git a/blog/tags/release/index.html b/blog/tags/release/index.html
index aa0236a3..3437d960 100644
--- a/blog/tags/release/index.html
+++ b/blog/tags/release/index.html
@@ -5,54 +5,131 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">One post tagged with "release" | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/blog/tags/release"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsea [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.01bb8940.js" defer="defer"></script>
-<script src="/assets/js/main.0cc2e93a.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.8489d39d.js" defer="defer"></script>
+<script src="/assets/js/main.4f5176d9.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
-<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
-<p>This article is the first of a four part series about federated data analysis using Apache Wayang.
-The first article starts with an introduction of a typical data colaboration scenario which will emerge in our digital future.</p>
-<p>In part two and three we will share a summary of our Apache Kafka client implementation for Apache Wayang.
-We started with the Java Platform (part 2) and the Apache Spark implementation follows (W.I.P.) in part three.</p>
-<p>The use case behind this work is an imaginary data collaboration scenario.
-We see this example and the demand for a solution already in many places.
-For us this is motivation enough to propose a solution.
-This would also allow us to do more local data processing, and businesses can stop moving data around the world, but rather care about data locality while they expose and share specific information to others by using data federation.
-This reduces complexity of data management and cost dramatically.</p>
-<p>For this purpose, we illustrate a cross organizational data sharing scenario from the finance sector soon.
-This analysis pattern will also be relevant in the context of data analysis along supply chains, another typical example where data from many stakeholder together is needed but never managed in one place, for good reasons.</p>
-<p>Data federation can help us to unlock the hidden value of all those isolated data lakes.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="a-cross-organizational-data-sharing-scenario">A cross organizational data sharing scenario<a href="#a-cross-organizational-data-sharing-scenario" class="hash-link" aria-label="Direct link to A cross organizational data sharing scenario" title="Direct link to A cross organizational data sharing scenario"></a></h2>
-<p>Our goal is the implementation of a cross organization decentralized data processing scenario, in which protected local data should be processed in combination with public data from public sources in a collaborative manner.
-Instead of copying all data into a central data lake or a central data platform we decided to use federated analytics.
-Apache Wayang is the tool we work with.
-In our case, the public data is hosted on publicly available websites or data pods.
-A client can use the HTTP(S) protocol to read the data which is given in a well defined format.
-For simplicity we decided to use CSV format.
-When we look into the data of each participant we have a different perspective.</p>
-<p>Our processing procedure should calculate a particular metric on the <em>local data</em> of each participant.
-An example of such a metric is the average spending of all users on a particular product category per month.
-This can vary from partner to partner, hence, we want to be able to calculate a peer-group comparison so that each partner can see its own metric compared with a global average calculated from contributions by all partners.
-Such a process requires global averaging and local averaging.
-And due to governance constraints, we can’t bring all raw data together in one place.</p>
-<p>Instead, we want to use Apache Wayang for this purpose.
-We simplify the procedure and split it into two phases.
-Phase one is the process, which allows each participant to calculate the local metrics.
-This requires only local data. The second phase requires data from all collaborating partners.
-The monthly sum and counter values per partner and category are needed in one place by all other parties.
-Hence, the algorithm of the first phase stores the local results locally, and the contributions to the global results in an externally accessible Kafka topic.
-We assume this is done by each of the partners.</p>
-<p>Now we have a scenario, in which an Apache Wayang process must be able to read data from multiple Apache Kafka topics from multiple Apache Kafka clusters but finally writes into a single Kafka topic, which then can be accessed by all the participating clients.</p>
-<p><img decoding="async" loading="lazy" alt="images/image-1.png" src="/assets/images/image-1-9cc35d5aea2b867d7e5759a96bd02334.png" width="904" height="550" class="img_ev3q"></p>
-<p>The illustration shows the data flows in such a scenario.
-Jobs with red border are executed by the participants in isolation within their own data processing environments.
-But they share some of the data, using publicly accessible Kafka topics, marked by A. Job 4 is the Apache Wayang job in our focus: here we intent to read data from 3 different source systems, and write results into a fourth system (marked as B), which can be accesses by all participants again.</p>
-<p>With this in mind we want to implement an Apache Wayang application which implements the illustrated <em>Job 4</em>.
-Since as of today, there is now <em>KafkaSource</em> and <em>KafkaSink</em> available in Apache Wayang, an implementation of both will be our first step.
-Our assumption is, that in the beginning, there won’t be much data.</p>
-<p>Apache Spark is not required to cope with the load, but we expect, that in the future, a single Java application would not be able to handle our workload.
-Hence, we want to utilize the Apache Wayang abstraction over multiple processing platforms, starting with Java.
-Later, we want to switch to Apache Spark.</p></div><footer class="row docusaurus-mt-lg"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/kafka">kafka</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/cross-organization-data-collaboration">cross organization [...]
+<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
+<p>The ASF provides a robust infrastructure for open communities of software developers. We can share ideas, combine forces, contribute code, docs, review-energy, art work, and from time to time we can nail it down. A release defines an intermediate result of the continuous community work.</p>
+<p>How we do such a release in the Apache Wayang team is an essential aspect towards graduation. First of all, there are some references to take into account, such as:</p>
+<ul>
+<li><a href="https://www.apache.org/legal/release-policy.html" target="_blank" rel="noopener noreferrer">https://www.apache.org/legal/release-policy.html</a></li>
+<li><a href="https://maven.apache.org/maven-release/maven-release-plugin/" target="_blank" rel="noopener noreferrer">https://maven.apache.org/maven-release/maven-release-plugin/</a></li>
+</ul>
+<p>Assuming you are (P)PMC, and assuming that you have the right permissions for such a release, you can follow the path as described in this guide:</p>
+<ul>
+<li><a href="https://plc4x.apache.org/developers/release/release.html" target="_blank" rel="noopener noreferrer">https://plc4x.apache.org/developers/release/release.html</a></li>
+</ul>
+<p>I tried to follow exactly this procedure, several times. I failed. Here I share the current status of my <strong>release attempts</strong>.</p>
+<p>I plan a longer tour, and do not want to block the project for a long time.
+Hence I create this draft, and I hope we can unblock this project as soon as possible.</p>
+<h2 class="anchor anchorWithStickyNavbar_LWe7" id="status">Status:<a href="#status" class="hash-link" aria-label="Direct link to Status:" title="Direct link to Status:"></a></h2>
+<p>I am not able to conduct the <em>mvn release:perform</em> step.
+Anything before worked, sometimes only after some digging, but it worked.</p>
+<ul>
+<li>
+<p>We assume, that due to my membership in two ASF incubator projects I am not able to upload the artefacts to the Nexus repository (H1).</p>
+</li>
+<li>
+<p>It can be, that I have not the correct user and password in my <em>settings.xml</em> file (H2).</p>
+</li>
+</ul>
+<p>But I tested a manual login to the nexus server <a href="https://repository.apache.org/service/local/staging/deploy/maven2" target="_blank" rel="noopener noreferrer">https://repository.apache.org/service/local/staging/deploy/maven2</a> with success. And beyond that I have no idea how I can verify this detail alone.</p>
+<h2 class="anchor anchorWithStickyNavbar_LWe7" id="idea--proposal">Idea / Proposal<a href="#idea--proposal" class="hash-link" aria-label="Direct link to Idea / Proposal" title="Direct link to Idea / Proposal"></a></h2>
+<p>(1) It would be great, if someone - who has done a release in any other ASF project or in Apache Wayang - could follow the steps I share, so that we can check where the problem hides itself.</p>
+<p>(2) As a follow-up task, I suggest to add a <strong>Release Guide</strong> to the Apache Wayang project, including release manager onboarding steps, and checklists for the particular project, derived from the referenced sources which are listed above.</p>
+<p>But for now it is all about sharing the status (as I did serveral times on multiple chanels, including JIRA, Slack, Mailing lists) and finding a solution for Apache Wayang release 1.0.</p>
+<h2 class="anchor anchorWithStickyNavbar_LWe7" id="latest-error">Latest Error:<a href="#latest-error" class="hash-link" aria-label="Direct link to Latest Error:" title="Direct link to Latest Error:"></a></h2>
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn release:perform -X -DskipTests</span><br></span></code></pre><div class="buttonGroup__atx"><button ty [...]
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">[INFO] Caused by: org.eclipse.aether.deployment.DeploymentException: Failed to deploy artifacts: Could no [...]
+<h2 class="anchor anchorWithStickyNavbar_LWe7" id="activity-log">Activity Log<a href="#activity-log" class="hash-link" aria-label="Direct link to Activity Log" title="Direct link to Activity Log"></a></h2>
+<div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn release:clean</span><br></span><span class="token-line" style="color:#393A34"><span cla [...]
+<div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn release:prepare </span><span class="token parameter variable" style="color:#36acaa">-Da [...]
+<div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token punctuation" style="color:#393A34">[</span><span class="token plain">ERROR</span><span class="toke [...]
+<div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn versions:set </span><span class="token parameter variable" style="color:#36acaa">-DnewV [...]
+<div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn release:prepare </span><span class="token parameter variable" style="color:#36acaa">-Da [...]
+<div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token punctuation" style="color:#393A34">[</span><span class="token plain">ERROR</span><span class="toke [...]
+<div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token function" style="color:#d73a49">git</span><span class="token plain"> status</span><br></span><span [...]
+<div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn release:prepare </span><span class="token parameter variable" style="color:#36acaa">-Da [...]
+<div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">Caused by: org.eclipse.aether.transfer.NoRepositoryConnectorException: Blocked mirror </spa [...]
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="dependency-on-jdk-11-during-release">Dependency on JDK-11 during release<a href="#dependency-on-jdk-11-during-release" class="hash-link" aria-label="Direct link to Dependency on JDK-11 during release" title="Direct link to Dependency on JDK-11 during release"></a></h3>
+<blockquote>
+<p>FIXED with local JDK11 Setup</p>
+</blockquote>
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">brew install openjdk@11</span><br></span><span class="token-line" style="color:#393A34"><span class="toke [...]
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="manual-update-of-release-version">Manual update of release-version<a href="#manual-update-of-release-version" class="hash-link" aria-label="Direct link to Manual update of release-version" title="Direct link to Manual update of release-version"></a></h3>
+<p>During the release procedure, do I have to set the version here in this configuration section manually?</p>
+<div class="language-xml codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-xml codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token tag punctuation" style="color:#393A34"><</span><span class="token tag" style="color:#00009f">plug [...]
+<blockquote>
+<p>It seems that these properties must be updated manually.</p>
+</blockquote>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="warning-regarding-illegal-reflective-access-operation">Warning regarding "illegal reflective access operation"<a href="#warning-regarding-illegal-reflective-access-operation" class="hash-link" aria-label="Direct link to Warning regarding "illegal reflective access operation"" title="Direct link to Warning regarding "illegal reflective access operation""></a></h3>
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">[ERROR] WARNING: An illegal reflective access operation has occurred</span><br></span><span class="token- [...]
+<blockquote>
+<p><strong>This is still an OPEN ISSUE !</strong></p>
+</blockquote>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="rat-check-fails">RAT Check fails<a href="#rat-check-fails" class="hash-link" aria-label="Direct link to RAT Check fails" title="Direct link to RAT Check fails"></a></h3>
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">[INFO] [ERROR] Failed to execute goal org.apache.rat:apache-rat-plugin:0.13:check (license-check) on proj [...]
+<div class="language-cat codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-cat codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#393A34" [...]
+<blockquote>
+<p><em>FIXED by adding .java-versions to .gitignore</em></p>
+</blockquote>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="tag-could-not-be-created-in-scm">Tag could not be created in SCM.<a href="#tag-could-not-be-created-in-scm" class="hash-link" aria-label="Direct link to Tag could not be created in SCM." title="Direct link to Tag could not be created in SCM."></a></h3>
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">[ERROR] Failed to execute goal org.apache.maven.plugins:maven-release-plugin:3.0.1:prepare (default-cli) [...]
+<blockquote>
+<p>FIXED by manual changes in pom.xml.</p>
+</blockquote>
+<div class="language-xml codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-xml codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#393A34" [...]
+<div class="language-xml codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-xml codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token tag punctuation" style="color:#393A34"><</span><span class="token tag" style="color:#00009f">conf [...]
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn release:prepare -Darguments='-DskipTests=True -Dresume=True'</span><br></span></code></pre> [...]
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn clean package</span><br></span><span class="token-line" style="color:#393A34"><span class="token plai [...]
+<blockquote>
+<p>So far so good. But now the sun went down.</p>
+</blockquote>
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">[INFO] [ERROR] Failed to execute goal org.apache.maven.plugins:maven-deploy-plugin:3.0.0-M1:deploy (defau [...]
+<p>The null pointer exception indicates problems in <em>settings.xml</em>. This could be fixed.
+But now, the error changes with and I am still not able to stage the build results.</p>
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn release:perform -X -DskipTests</span><br></span></code></pre><div class="buttonGroup__atx"><button ty [...]
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">[INFO] Caused by: org.eclipse.aether.deployment.DeploymentException: Failed to deploy artifacts: Could no [...]
+<h2 class="anchor anchorWithStickyNavbar_LWe7" id="checklist-for-next-iteration">Checklist for next iteration:<a href="#checklist-for-next-iteration" class="hash-link" aria-label="Direct link to Checklist for next iteration:" title="Direct link to Checklist for next iteration:"></a></h2>
+<p>The error you're encountering indicates that there is a problem with deploying the artifacts using the <code>maven-deploy-plugin</code>. The root cause of the error is a <code>NullPointerException</code> during the deployment process, specifically related to the <code>org.eclipse.aether.transfer.ArtifactTransferException</code>.</p>
+<p>Here's a step-by-step guide to troubleshoot and resolve this issue:</p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="1-check-maven-settings">1. Check Maven Settings<a href="#1-check-maven-settings" class="hash-link" aria-label="Direct link to 1. Check Maven Settings" title="Direct link to 1. Check Maven Settings"></a></h3>
+<p>Ensure that your Maven settings (<code>settings.xml</code>) are correctly configured for deployment. Verify that the repository settings and credentials are correctly specified.</p>
+<p><strong>[DONE]</strong></p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="2-verify-repository-url">2. Verify Repository URL<a href="#2-verify-repository-url" class="hash-link" aria-label="Direct link to 2. Verify Repository URL" title="Direct link to 2. Verify Repository URL"></a></h3>
+<p>Make sure the repository URL in your <code>pom.xml</code> or <code>settings.xml</code> is correct and reachable. The URL should point to the correct staging repository for deployment.</p>
+<p><em>[OPEN]</em> - I did not touch it, so I guess it is correct in the <em>pom.xml</em>.</p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="3-maven-version-compatibility">3. Maven Version Compatibility<a href="#3-maven-version-compatibility" class="hash-link" aria-label="Direct link to 3. Maven Version Compatibility" title="Direct link to 3. Maven Version Compatibility"></a></h3>
+<p>Verify that you are using a compatible version of Maven. Sometimes, upgrading or downgrading Maven can resolve such issues.</p>
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">Apache Maven 3.9.6 (bc0240f3c744dd6b6ec2920b3cd08dcc295161ae)</span><br></span><span class="token-line" s [...]
+<p><strong>[DONE]</strong></p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="4-check-for-network-issues">4. Check for Network Issues<a href="#4-check-for-network-issues" class="hash-link" aria-label="Direct link to 4. Check for Network Issues" title="Direct link to 4. Check for Network Issues"></a></h3>
+<p>Ensure there are no network issues that might be causing problems in connecting to the repository. Sometimes, network configurations, firewalls, or proxy settings can interfere with the deployment process.</p>
+<p><strong>[DONE]</strong></p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="5-update-maven-plugins">5. Update Maven Plugins<a href="#5-update-maven-plugins" class="hash-link" aria-label="Direct link to 5. Update Maven Plugins" title="Direct link to 5. Update Maven Plugins"></a></h3>
+<p>Ensure that you are using the latest versions of the Maven plugins. Sometimes, bugs in older versions can cause unexpected issues.</p>
+<p><em>[OPEN]</em></p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="6-configure-the-maven-deploy-plugin-in-pomxml">6. Configure the <code>maven-deploy-plugin</code> in <code>pom.xml</code><a href="#6-configure-the-maven-deploy-plugin-in-pomxml" class="hash-link" aria-label="Direct link to 6-configure-the-maven-deploy-plugin-in-pomxml" title="Direct link to 6-configure-the-maven-deploy-plugin-in-pomxml"></a></h3>
+<p>Make sure the <code>maven-deploy-plugin</code> is correctly configured in your <code>pom.xml</code>. Here's an example configuration:</p>
+<div class="language-xml codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-xml codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token tag punctuation" style="color:#393A34"><</span><span class="token tag" style="color:#00009f">plug [...]
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="7-increase-verbose-logging">7. Increase Verbose Logging<a href="#7-increase-verbose-logging" class="hash-link" aria-label="Direct link to 7. Increase Verbose Logging" title="Direct link to 7. Increase Verbose Logging"></a></h3>
+<p>Enable verbose logging to get more details about the error. You can do this by adding the <code>-X</code> flag when running the Maven command:</p>
+<div class="language-sh codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-sh codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn clean deploy </span><span class="token parameter variable" style="color:#36acaa">-X</span>< [...]
+<p><strong>[DONE]</strong></p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="8-retry-with-a-clean-local-repository">8. Retry with a Clean Local Repository<a href="#8-retry-with-a-clean-local-repository" class="hash-link" aria-label="Direct link to 8. Retry with a Clean Local Repository" title="Direct link to 8. Retry with a Clean Local Repository"></a></h3>
+<p>Sometimes, a corrupt local repository can cause issues. Try cleaning your local Maven repository and re-running the deployment:</p>
+<div class="language-sh codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-sh codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn clean </span><span class="token function" style="color:#d73a49">install</span><span class=" [...]
+<p><em>[OPEN]</em> - Does not match to an authentication issue.</p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="9-check-for-missing-credentials">9. Check for Missing Credentials<a href="#9-check-for-missing-credentials" class="hash-link" aria-label="Direct link to 9. Check for Missing Credentials" title="Direct link to 9. Check for Missing Credentials"></a></h3>
+<p>Ensure that the credentials for the repository are correctly set up in your <code>settings.xml</code>:</p>
+<div class="language-xml codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-xml codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token tag punctuation" style="color:#393A34"><</span><span class="token tag" style="color:#00009f">serv [...]
+<p><strong>[DONE]</strong></p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="10-review-the-full-stack-trace">10. Review the Full Stack Trace<a href="#10-review-the-full-stack-trace" class="hash-link" aria-label="Direct link to 10. Review the Full Stack Trace" title="Direct link to 10. Review the Full Stack Trace"></a></h3>
+<p>The full stack trace indicates a <code>NullPointerException</code>:</p>
+<div class="language-plaintext codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-plaintext codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">Caused by: java.lang.NullPointerException</span><br></span><span class="token-lin [...]
+<p>This suggests that there might be an issue with how credentials are being handled. Double-check that the credentials are being correctly passed and processed.</p>
+<p><strong>[DONE]</strong></p></div><footer class="row docusaurus-mt-lg"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/asf">ASF</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/release">release</a></li></ul></div></footer></article><nav class="paginatio [...]
<p> Apache Wayang is an effort undergoing incubation at The Apache Software Foundation (ASF), sponsored by the Apache Incubator. Incubation is required of all newly accepted projects until a further review indicates that the infrastructure, communications, and decision making process have stabilized in a manner consistent with other successful ASF projects. While incubation status is not necessarily a reflection of the completeness or stability of the code, it does indicate that th [...]
<p>
Copyright © 2024 The Apache Software Foundation, Licensed under the Apache License, Version 2.0. <br>
diff --git a/blog/tags/spark/index.html b/blog/tags/spark/index.html
index 0491b958..5861cb0e 100644
--- a/blog/tags/spark/index.html
+++ b/blog/tags/spark/index.html
@@ -3,70 +3,13 @@
<head>
<meta charset="UTF-8">
<meta name="generator" content="Docusaurus v3.1.1">
-<title data-rh="true">2 posts tagged with "spark" | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/blog/tags/spark"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:l [...]
+<title data-rh="true">One post tagged with "spark" | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/blog/tags/spark"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch: [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.01bb8940.js" defer="defer"></script>
-<script src="/assets/js/main.0cc2e93a.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.8489d39d.js" defer="defer"></script>
+<script src="/assets/js/main.4f5176d9.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
-<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
-We look into the “Read- and Write-Path” for our data items, called <em>DataQuanta</em>.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="apache-wayangs-read--write-path-for-kafka-topics">Apache Wayang’s Read & Write Path for Kafka topics<a href="#apache-wayangs-read--write-path-for-kafka-topics" class="hash-link" aria-label="Direct link to Apache Wayang’s Read & Write Path for Kafka topics" title="Direct link to Apache Wayang’s Read & Write Path for Kafka topics"></a></h2>
-<p>To describe the read and write paths for data in the context of the created Apache Wayang code snippet, the primary classes and interfaces we need to understand are as follows:</p>
-<p><strong>WayangContext:</strong> This class is essential for initializing the Wayang processing environment.
-It allows you to configure the execution environment and register plugins that define which platforms Wayang can use for data processing tasks, such as <em>Java.basicPlugin()</em> for local Java execution.</p>
-<p><strong>JavaPlanBuilder:</strong> This class is used to build and define the data processing pipeline (or plan) in Wayang.
-It provides a fluent API to specify the operations to be performed on the data, from reading the input to processing it and writing the output.</p>
-<h3 class="anchor anchorWithStickyNavbar_LWe7" id="read-path">Read Path<a href="#read-path" class="hash-link" aria-label="Direct link to Read Path" title="Direct link to Read Path"></a></h3>
-<p>The read path describes how data is ingested from a source into the Wayang processing pipeline:</p>
-<p><em>Reading from Kafka Topic:</em> The method <em>readKafkaTopic(topicName)</em> is used to ingest data from a specified Kafka topic.
-This is the starting point of the data processing pipeline, where topicName represents the name of the Kafka topic from which data is read.</p>
-<p><em>Data Tokenization and Preparation:</em> Once the data is read from Kafka, it undergoes several transformations such as Splitting, Filtering, and Mapping.
-What follows are the procedures known as Reducing, Grouping, Co-Grouping, and Counting.</p>
-<h3 class="anchor anchorWithStickyNavbar_LWe7" id="write-path">Write Path<a href="#write-path" class="hash-link" aria-label="Direct link to Write Path" title="Direct link to Write Path"></a></h3>
-<p><em>Writing to Kafka Topic:</em> The final step in the pipeline involves writing the processed data back to a Kafka topic using <em>.writeKafkaTopic(...)</em>.
-This method takes parameters that specify the target Kafka topic, a serialization function to format the data as strings, and additional configuration for load profile estimation, which optimizes the writing process.</p>
-<p>This read-write path provides a comprehensive flow of data from ingestion from Kafka, through various processing steps, and finally back to Kafka, showcasing a full cycle of data processing within Apache Wayang's abstracted environment and is implemented in our example program shown in <em>listing 1</em>.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="implementation-of-input--and-output-operators">Implementation of Input- and Output Operators<a href="#implementation-of-input--and-output-operators" class="hash-link" aria-label="Direct link to Implementation of Input- and Output Operators" title="Direct link to Implementation of Input- and Output Operators"></a></h2>
-<p>The next section shows how a new pair of operators can be implemented to extend Apache Wayang’s capabilities on the input and output side.
-We created the Kafka Source and Kafka Sink components so that our cross organizational data collaboration scenario can be implemented using data streaming infrastructure.</p>
-<p><strong>Level 1 – Wayang execution plan with abstract operators</strong></p>
-<p>The implementation of our Kafka Source and Kafka Sink components for Apache Wayang requires new methods and classes on three layers.
-First of all in the API package.
-Here we use the JavaPlanBuilder to expose the function for selecting a Kafka topic as the source to be used by client.
-The class <em>JavaPlanBuilder</em> in package <em>org.apache.wayang.api</em> in the project <em>wayang-api/wayang-api-scala-java</em> exposes our new functionality to our external client.
-An instance of the JavaPlanBuilder is used to define the data processing pipeline.
-We use its <em>readKafkaTopic()</em> which specifies the source Kafka topic to read from, and for the write path we use the <em>writeKafkaTopic()</em> method.
-Both Methods do only trigger activities in the background.</p>
-<p>For the output side, we use the <em>DataQuantaBuilder</em> class, which offers an implementation of the writeKafkaTopic function.
-This function is designed to send processed data, referred to as DataQuanta, to a specified Kafka topic.
-Essentially, it marks the final step in a data processing sequence constructed using the Apache Wayang framework.</p>
-<p>In the DataQuanta class we implemented the methods writeKafkaTopic and writeKafkaTopicJava which use the KafkaTopicSink class.
-In this API layer we use the Scala programming language, but we utilize the Java classes, implemented in the layer below.</p>
-<p><strong>Level 2 – Wiring between Platform Abstraction and Implementation</strong></p>
-<p>The second layer builds the bridge between the WayangContext and PlanBuilders which work together with DataQuanta and the DataQuantaBuilder.</p>
-<p>Also, the mapping between the abstract components and the specific implementations are defined in this layer.</p>
-<p>Therefore, the mappings package has a class <em>Mappings</em> in which all relevant input and output operators are listed.
-We use it to register the KafkaSourceMapping and a KafkaSinkMapping for the particular platform, Java in our case.
-These classes allow the Apache Wayang framework to use the Java implementation of the KafkaTopicSource component (and KafkaTopicSink respectively).
-While the Wayang execution plan uses the higher abstractions, here on the “platform level” we have to link the specific implementation for the target platform.
-In our case this leads to a Java program running on a JVM which is set up by the Apache Wayang framework using the logical components of the execution plan.</p>
-<p>Those mappings link the real implementation of our operators the ones used in an execution plan.
-The JavaKafkaTopicSource and the JavaKafkaTopicSink extend the KafkaTopicSource and KafkaTopicSink so that the lower level implementation of those classes become available within Wayang’s Java Platform context.</p>
-<p>In this layer, the KafkaConsumer class and the KafkaProducer class are used, but both are configured and instantiated in the next layer underneath.
-All this is done in the project <em>wayang-plarforms/wayang-java</em>.</p>
-<p><strong>Layer 3 – Input/Output Connector Layer</strong></p>
-<p>The <em>KafkaTopicSource</em> and <em>KafkaTopicSink</em> classes build the third layer of our implementation.
-Both are implemented in Java programming language.
-In this layer, the real Kafka-Client logic is defined.
-Details about consumer and producers, client configuration, and schema handling have to be handled here.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="summary">Summary<a href="#summary" class="hash-link" aria-label="Direct link to Summary" title="Direct link to Summary"></a></h2>
-<p>Both classes in the third layer implement the Kafka client logic which is needed by the Wayang-execution plan when external data flows should be established.
-The layer above handles the mapping of the components at startup time.
-All this wiring is needed to keep Wayang open and flexible so that multiple external systems can be used in a variety of combinations and using multiple target platforms in combinations.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="outlook">Outlook<a href="#outlook" class="hash-link" aria-label="Direct link to Outlook" title="Direct link to Outlook"></a></h2>
-<p>The next part of the article series will cover the creation of an Kafka Source and Sink component for the Apache Spark platform, which allows our use case to scale.
-Finally, in part four we bring all puzzles together, and show the full implementation of the multi organizational data collaboration use case.</p></div><footer class="row docusaurus-mt-lg"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/kafka">kafka</a></li><li class="tag_QGVx"><a class= [...]
+<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
Motivated by the learnings from last time, I stated implementing a Kafka Source component and a Kafka Sink component for the Apache Spark platform in Apache Wayang.
In our previous article we shared the results of the work on the frist Apache Kafka integration using the Java Platform.</p>
<p>Let's see how it goes this time with Apache Spark.</p>
diff --git a/blog/tags/tensorflow/index.html b/blog/tags/tensorflow/index.html
index 11683a6b..ea05193a 100644
--- a/blog/tags/tensorflow/index.html
+++ b/blog/tags/tensorflow/index.html
@@ -5,11 +5,11 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">One post tagged with "tensorflow" | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/blog/tags/tensorflow"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name=" [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.01bb8940.js" defer="defer"></script>
-<script src="/assets/js/main.0cc2e93a.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.8489d39d.js" defer="defer"></script>
+<script src="/assets/js/main.4f5176d9.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
-<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
+<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
Thanks to the extensible nature of Wayang, the only core changes we had to do were introducing the concept of a <code>Model</code> and implement a new driver for the newly added platform.</p>
<h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-1-introducing-a-model">Step 1: Introducing a Model<a href="#step-1-introducing-a-model" class="hash-link" aria-label="Direct link to Step 1: Introducing a Model" title="Direct link to Step 1: Introducing a Model"></a></h2>
<p>With respect to the model, we followed Wayang’s abstraction philosophy: We created a <code>Model</code> interface to be used as input or output by Wayang operators and then extended it for the platform-specific operators. Different model interfaces can be found here:</p>
diff --git a/blog/tags/trino/index.html b/blog/tags/trino/index.html
index daef3708..563d457c 100644
--- a/blog/tags/trino/index.html
+++ b/blog/tags/trino/index.html
@@ -5,11 +5,11 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">One post tagged with "trino" | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/blog/tags/trino"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch: [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.01bb8940.js" defer="defer"></script>
-<script src="/assets/js/main.0cc2e93a.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.8489d39d.js" defer="defer"></script>
+<script src="/assets/js/main.4f5176d9.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
-<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
+<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
<p> Apache Wayang is an effort undergoing incubation at The Apache Software Foundation (ASF), sponsored by the Apache Incubator. Incubation is required of all newly accepted projects until a further review indicates that the infrastructure, communications, and decision making process have stabilized in a manner consistent with other successful ASF projects. While incubation status is not necessarily a reflection of the completeness or stability of the code, it does indicate that th [...]
<p>
Copyright © 2024 The Apache Software Foundation, Licensed under the Apache License, Version 2.0. <br>
diff --git a/blog/tags/wayang/index.html b/blog/tags/wayang/index.html
index 7a278509..1b595dde 100644
--- a/blog/tags/wayang/index.html
+++ b/blog/tags/wayang/index.html
@@ -3,213 +3,133 @@
<head>
<meta charset="UTF-8">
<meta name="generator" content="Docusaurus v3.1.1">
-<title data-rh="true">12 posts tagged with "wayang" | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/blog/tags/wayang"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearc [...]
+<title data-rh="true">9 posts tagged with "wayang" | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/blog/tags/wayang"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.01bb8940.js" defer="defer"></script>
-<script src="/assets/js/main.0cc2e93a.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.8489d39d.js" defer="defer"></script>
+<script src="/assets/js/main.4f5176d9.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
-<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
-<p>This article is the first of a four part series about federated data analysis using Apache Wayang.
-The first article starts with an introduction of a typical data colaboration scenario which will emerge in our digital future.</p>
-<p>In part two and three we will share a summary of our Apache Kafka client implementation for Apache Wayang.
-We started with the Java Platform (part 2) and the Apache Spark implementation follows (W.I.P.) in part three.</p>
-<p>The use case behind this work is an imaginary data collaboration scenario.
-We see this example and the demand for a solution already in many places.
-For us this is motivation enough to propose a solution.
-This would also allow us to do more local data processing, and businesses can stop moving data around the world, but rather care about data locality while they expose and share specific information to others by using data federation.
-This reduces complexity of data management and cost dramatically.</p>
-<p>For this purpose, we illustrate a cross organizational data sharing scenario from the finance sector soon.
-This analysis pattern will also be relevant in the context of data analysis along supply chains, another typical example where data from many stakeholder together is needed but never managed in one place, for good reasons.</p>
-<p>Data federation can help us to unlock the hidden value of all those isolated data lakes.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="a-cross-organizational-data-sharing-scenario">A cross organizational data sharing scenario<a href="#a-cross-organizational-data-sharing-scenario" class="hash-link" aria-label="Direct link to A cross organizational data sharing scenario" title="Direct link to A cross organizational data sharing scenario"></a></h2>
-<p>Our goal is the implementation of a cross organization decentralized data processing scenario, in which protected local data should be processed in combination with public data from public sources in a collaborative manner.
-Instead of copying all data into a central data lake or a central data platform we decided to use federated analytics.
-Apache Wayang is the tool we work with.
-In our case, the public data is hosted on publicly available websites or data pods.
-A client can use the HTTP(S) protocol to read the data which is given in a well defined format.
-For simplicity we decided to use CSV format.
-When we look into the data of each participant we have a different perspective.</p>
-<p>Our processing procedure should calculate a particular metric on the <em>local data</em> of each participant.
-An example of such a metric is the average spending of all users on a particular product category per month.
-This can vary from partner to partner, hence, we want to be able to calculate a peer-group comparison so that each partner can see its own metric compared with a global average calculated from contributions by all partners.
-Such a process requires global averaging and local averaging.
-And due to governance constraints, we can’t bring all raw data together in one place.</p>
-<p>Instead, we want to use Apache Wayang for this purpose.
-We simplify the procedure and split it into two phases.
-Phase one is the process, which allows each participant to calculate the local metrics.
-This requires only local data. The second phase requires data from all collaborating partners.
-The monthly sum and counter values per partner and category are needed in one place by all other parties.
-Hence, the algorithm of the first phase stores the local results locally, and the contributions to the global results in an externally accessible Kafka topic.
-We assume this is done by each of the partners.</p>
-<p>Now we have a scenario, in which an Apache Wayang process must be able to read data from multiple Apache Kafka topics from multiple Apache Kafka clusters but finally writes into a single Kafka topic, which then can be accessed by all the participating clients.</p>
-<p><img decoding="async" loading="lazy" alt="images/image-1.png" src="/assets/images/image-1-9cc35d5aea2b867d7e5759a96bd02334.png" width="904" height="550" class="img_ev3q"></p>
-<p>The illustration shows the data flows in such a scenario.
-Jobs with red border are executed by the participants in isolation within their own data processing environments.
-But they share some of the data, using publicly accessible Kafka topics, marked by A. Job 4 is the Apache Wayang job in our focus: here we intent to read data from 3 different source systems, and write results into a fourth system (marked as B), which can be accesses by all participants again.</p>
-<p>With this in mind we want to implement an Apache Wayang application which implements the illustrated <em>Job 4</em>.
-Since as of today, there is now <em>KafkaSource</em> and <em>KafkaSink</em> available in Apache Wayang, an implementation of both will be our first step.
-Our assumption is, that in the beginning, there won’t be much data.</p>
-<p>Apache Spark is not required to cope with the load, but we expect, that in the future, a single Java application would not be able to handle our workload.
-Hence, we want to utilize the Apache Wayang abstraction over multiple processing platforms, starting with Java.
-Later, we want to switch to Apache Spark.</p></div><footer class="row docusaurus-mt-lg"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/kafka">kafka</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/cross-organization-data-collaboration">cross organization [...]
-We look into the “Read- and Write-Path” for our data items, called <em>DataQuanta</em>.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="apache-wayangs-read--write-path-for-kafka-topics">Apache Wayang’s Read & Write Path for Kafka topics<a href="#apache-wayangs-read--write-path-for-kafka-topics" class="hash-link" aria-label="Direct link to Apache Wayang’s Read & Write Path for Kafka topics" title="Direct link to Apache Wayang’s Read & Write Path for Kafka topics"></a></h2>
-<p>To describe the read and write paths for data in the context of the created Apache Wayang code snippet, the primary classes and interfaces we need to understand are as follows:</p>
-<p><strong>WayangContext:</strong> This class is essential for initializing the Wayang processing environment.
-It allows you to configure the execution environment and register plugins that define which platforms Wayang can use for data processing tasks, such as <em>Java.basicPlugin()</em> for local Java execution.</p>
-<p><strong>JavaPlanBuilder:</strong> This class is used to build and define the data processing pipeline (or plan) in Wayang.
-It provides a fluent API to specify the operations to be performed on the data, from reading the input to processing it and writing the output.</p>
-<h3 class="anchor anchorWithStickyNavbar_LWe7" id="read-path">Read Path<a href="#read-path" class="hash-link" aria-label="Direct link to Read Path" title="Direct link to Read Path"></a></h3>
-<p>The read path describes how data is ingested from a source into the Wayang processing pipeline:</p>
-<p><em>Reading from Kafka Topic:</em> The method <em>readKafkaTopic(topicName)</em> is used to ingest data from a specified Kafka topic.
-This is the starting point of the data processing pipeline, where topicName represents the name of the Kafka topic from which data is read.</p>
-<p><em>Data Tokenization and Preparation:</em> Once the data is read from Kafka, it undergoes several transformations such as Splitting, Filtering, and Mapping.
-What follows are the procedures known as Reducing, Grouping, Co-Grouping, and Counting.</p>
-<h3 class="anchor anchorWithStickyNavbar_LWe7" id="write-path">Write Path<a href="#write-path" class="hash-link" aria-label="Direct link to Write Path" title="Direct link to Write Path"></a></h3>
-<p><em>Writing to Kafka Topic:</em> The final step in the pipeline involves writing the processed data back to a Kafka topic using <em>.writeKafkaTopic(...)</em>.
-This method takes parameters that specify the target Kafka topic, a serialization function to format the data as strings, and additional configuration for load profile estimation, which optimizes the writing process.</p>
-<p>This read-write path provides a comprehensive flow of data from ingestion from Kafka, through various processing steps, and finally back to Kafka, showcasing a full cycle of data processing within Apache Wayang's abstracted environment and is implemented in our example program shown in <em>listing 1</em>.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="implementation-of-input--and-output-operators">Implementation of Input- and Output Operators<a href="#implementation-of-input--and-output-operators" class="hash-link" aria-label="Direct link to Implementation of Input- and Output Operators" title="Direct link to Implementation of Input- and Output Operators"></a></h2>
-<p>The next section shows how a new pair of operators can be implemented to extend Apache Wayang’s capabilities on the input and output side.
-We created the Kafka Source and Kafka Sink components so that our cross organizational data collaboration scenario can be implemented using data streaming infrastructure.</p>
-<p><strong>Level 1 – Wayang execution plan with abstract operators</strong></p>
-<p>The implementation of our Kafka Source and Kafka Sink components for Apache Wayang requires new methods and classes on three layers.
-First of all in the API package.
-Here we use the JavaPlanBuilder to expose the function for selecting a Kafka topic as the source to be used by client.
-The class <em>JavaPlanBuilder</em> in package <em>org.apache.wayang.api</em> in the project <em>wayang-api/wayang-api-scala-java</em> exposes our new functionality to our external client.
-An instance of the JavaPlanBuilder is used to define the data processing pipeline.
-We use its <em>readKafkaTopic()</em> which specifies the source Kafka topic to read from, and for the write path we use the <em>writeKafkaTopic()</em> method.
-Both Methods do only trigger activities in the background.</p>
-<p>For the output side, we use the <em>DataQuantaBuilder</em> class, which offers an implementation of the writeKafkaTopic function.
-This function is designed to send processed data, referred to as DataQuanta, to a specified Kafka topic.
-Essentially, it marks the final step in a data processing sequence constructed using the Apache Wayang framework.</p>
-<p>In the DataQuanta class we implemented the methods writeKafkaTopic and writeKafkaTopicJava which use the KafkaTopicSink class.
-In this API layer we use the Scala programming language, but we utilize the Java classes, implemented in the layer below.</p>
-<p><strong>Level 2 – Wiring between Platform Abstraction and Implementation</strong></p>
-<p>The second layer builds the bridge between the WayangContext and PlanBuilders which work together with DataQuanta and the DataQuantaBuilder.</p>
-<p>Also, the mapping between the abstract components and the specific implementations are defined in this layer.</p>
-<p>Therefore, the mappings package has a class <em>Mappings</em> in which all relevant input and output operators are listed.
-We use it to register the KafkaSourceMapping and a KafkaSinkMapping for the particular platform, Java in our case.
-These classes allow the Apache Wayang framework to use the Java implementation of the KafkaTopicSource component (and KafkaTopicSink respectively).
-While the Wayang execution plan uses the higher abstractions, here on the “platform level” we have to link the specific implementation for the target platform.
-In our case this leads to a Java program running on a JVM which is set up by the Apache Wayang framework using the logical components of the execution plan.</p>
-<p>Those mappings link the real implementation of our operators the ones used in an execution plan.
-The JavaKafkaTopicSource and the JavaKafkaTopicSink extend the KafkaTopicSource and KafkaTopicSink so that the lower level implementation of those classes become available within Wayang’s Java Platform context.</p>
-<p>In this layer, the KafkaConsumer class and the KafkaProducer class are used, but both are configured and instantiated in the next layer underneath.
-All this is done in the project <em>wayang-plarforms/wayang-java</em>.</p>
-<p><strong>Layer 3 – Input/Output Connector Layer</strong></p>
-<p>The <em>KafkaTopicSource</em> and <em>KafkaTopicSink</em> classes build the third layer of our implementation.
-Both are implemented in Java programming language.
-In this layer, the real Kafka-Client logic is defined.
-Details about consumer and producers, client configuration, and schema handling have to be handled here.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="summary">Summary<a href="#summary" class="hash-link" aria-label="Direct link to Summary" title="Direct link to Summary"></a></h2>
-<p>Both classes in the third layer implement the Kafka client logic which is needed by the Wayang-execution plan when external data flows should be established.
-The layer above handles the mapping of the components at startup time.
-All this wiring is needed to keep Wayang open and flexible so that multiple external systems can be used in a variety of combinations and using multiple target platforms in combinations.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="outlook">Outlook<a href="#outlook" class="hash-link" aria-label="Direct link to Outlook" title="Direct link to Outlook"></a></h2>
-<p>The next part of the article series will cover the creation of an Kafka Source and Sink component for the Apache Spark platform, which allows our use case to scale.
-Finally, in part four we bring all puzzles together, and show the full implementation of the multi organizational data collaboration use case.</p></div><footer class="row docusaurus-mt-lg"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/kafka">kafka</a></li><li class="tag_QGVx"><a class= [...]
-We look into the “Read- and Write-Path” for our data items, called <em>DataQuanta</em>.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="apache-wayangs-read--write-path-for-kafka-topics">Apache Wayang’s Read & Write Path for Kafka topics<a href="#apache-wayangs-read--write-path-for-kafka-topics" class="hash-link" aria-label="Direct link to Apache Wayang’s Read & Write Path for Kafka topics" title="Direct link to Apache Wayang’s Read & Write Path for Kafka topics"></a></h2>
-<p>To describe the read and write paths for data in the context of the created Apache Wayang code snippet, the primary classes and interfaces we need to understand are as follows:</p>
-<p><strong>WayangContext:</strong> This class is essential for initializing the Wayang processing environment.
-It allows you to configure the execution environment and register plugins that define which platforms Wayang can use for data processing tasks, such as <em>Java.basicPlugin()</em> for local Java execution.</p>
-<p><strong>JavaPlanBuilder:</strong> This class is used to build and define the data processing pipeline (or plan) in Wayang.
-It provides a fluent API to specify the operations to be performed on the data, from reading the input to processing it and writing the output.</p>
-<h3 class="anchor anchorWithStickyNavbar_LWe7" id="read-path">Read Path<a href="#read-path" class="hash-link" aria-label="Direct link to Read Path" title="Direct link to Read Path"></a></h3>
-<p>The read path describes how data is ingested from a source into the Wayang processing pipeline:</p>
-<p><em>Reading from Kafka Topic:</em> The method <em>readKafkaTopic(topicName)</em> is used to ingest data from a specified Kafka topic.
-This is the starting point of the data processing pipeline, where topicName represents the name of the Kafka topic from which data is read.</p>
-<p><em>Data Tokenization and Preparation:</em> Once the data is read from Kafka, it undergoes several transformations such as Splitting, Filtering, and Mapping.
-What follows are the procedures known as Reducing, Grouping, Co-Grouping, and Counting.</p>
-<h3 class="anchor anchorWithStickyNavbar_LWe7" id="write-path">Write Path<a href="#write-path" class="hash-link" aria-label="Direct link to Write Path" title="Direct link to Write Path"></a></h3>
-<p><em>Writing to Kafka Topic:</em> The final step in the pipeline involves writing the processed data back to a Kafka topic using <em>.writeKafkaTopic(...)</em>.
-This method takes parameters that specify the target Kafka topic, a serialization function to format the data as strings, and additional configuration for load profile estimation, which optimizes the writing process.</p>
-<p>This read-write path provides a comprehensive flow of data from ingestion from Kafka, through various processing steps, and finally back to Kafka, showcasing a full cycle of data processing within Apache Wayang's abstracted environment and is implemented in our example program shown in <em>listing 1</em>.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="implementation-of-input--and-output-operators">Implementation of Input- and Output Operators<a href="#implementation-of-input--and-output-operators" class="hash-link" aria-label="Direct link to Implementation of Input- and Output Operators" title="Direct link to Implementation of Input- and Output Operators"></a></h2>
-<p>The next section shows how a new pair of operators can be implemented to extend Apache Wayang’s capabilities on the input and output side.
-We created the Kafka Source and Kafka Sink components so that our cross organizational data collaboration scenario can be implemented using data streaming infrastructure.</p>
-<p><strong>Level 1 – Wayang execution plan with abstract operators</strong></p>
-<p>The implementation of our Kafka Source and Kafka Sink components for Apache Wayang requires new methods and classes on three layers.
-First of all in the API package.
-Here we use the JavaPlanBuilder to expose the function for selecting a Kafka topic as the source to be used by client.
-The class <em>JavaPlanBuilder</em> in package <em>org.apache.wayang.api</em> in the project <em>wayang-api/wayang-api-scala-java</em> exposes our new functionality to our external client.
-An instance of the JavaPlanBuilder is used to define the data processing pipeline.
-We use its <em>readKafkaTopic()</em> which specifies the source Kafka topic to read from, and for the write path we use the <em>writeKafkaTopic()</em> method.
-Both Methods do only trigger activities in the background.</p>
-<p>For the output side, we use the <em>DataQuantaBuilder</em> class, which offers an implementation of the writeKafkaTopic function.
-This function is designed to send processed data, referred to as DataQuanta, to a specified Kafka topic.
-Essentially, it marks the final step in a data processing sequence constructed using the Apache Wayang framework.</p>
-<p>In the DataQuanta class we implemented the methods writeKafkaTopic and writeKafkaTopicJava which use the KafkaTopicSink class.
-In this API layer we use the Scala programming language, but we utilize the Java classes, implemented in the layer below.</p>
-<p><strong>Level 2 – Wiring between Platform Abstraction and Implementation</strong></p>
-<p>The second layer builds the bridge between the WayangContext and PlanBuilders which work together with DataQuanta and the DataQuantaBuilder.</p>
-<p>Also, the mapping between the abstract components and the specific implementations are defined in this layer.</p>
-<p>Therefore, the mappings package has a class <em>Mappings</em> in which all relevant input and output operators are listed.
-We use it to register the KafkaSourceMapping and a KafkaSinkMapping for the particular platform, Java in our case.
-These classes allow the Apache Wayang framework to use the Java implementation of the KafkaTopicSource component (and KafkaTopicSink respectively).
-While the Wayang execution plan uses the higher abstractions, here on the “platform level” we have to link the specific implementation for the target platform.
-In our case this leads to a Java program running on a JVM which is set up by the Apache Wayang framework using the logical components of the execution plan.</p>
-<p>Those mappings link the real implementation of our operators the ones used in an execution plan.
-The JavaKafkaTopicSource and the JavaKafkaTopicSink extend the KafkaTopicSource and KafkaTopicSink so that the lower level implementation of those classes become available within Wayang’s Java Platform context.</p>
-<p>In this layer, the KafkaConsumer class and the KafkaProducer class are used, but both are configured and instantiated in the next layer underneath.
-All this is done in the project <em>wayang-plarforms/wayang-java</em>.</p>
-<p><strong>Layer 3 – Input/Output Connector Layer</strong></p>
-<p>The <em>KafkaTopicSource</em> and <em>KafkaTopicSink</em> classes build the third layer of our implementation.
-Both are implemented in Java programming language.
-In this layer, the real Kafka-Client logic is defined.
-Details about consumer and producers, client configuration, and schema handling have to be handled here.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="summary">Summary<a href="#summary" class="hash-link" aria-label="Direct link to Summary" title="Direct link to Summary"></a></h2>
-<p>Both classes in the third layer implement the Kafka client logic which is needed by the Wayang-execution plan when external data flows should be established.
-The layer above handles the mapping of the components at startup time.
-All this wiring is needed to keep Wayang open and flexible so that multiple external systems can be used in a variety of combinations and using multiple target platforms in combinations.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="outlook">Outlook<a href="#outlook" class="hash-link" aria-label="Direct link to Outlook" title="Direct link to Outlook"></a></h2>
-<p>The next part of the article series will cover the creation of an Kafka Source and Sink component for the Apache Spark platform, which allows our use case to scale.
-Finally, in part four we bring all puzzles together, and show the full implementation of the multi organizational data collaboration use case.</p></div><footer class="row docusaurus-mt-lg"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/kafka">kafka</a></li><li class="tag_QGVx"><a class= [...]
-<p>This article is the first of a four part series about federated data analysis using Apache Wayang.
-The first article starts with an introduction of a typical data colaboration scenario which will emerge in our digital future.</p>
-<p>In part two and three we will share a summary of our Apache Kafka client implementation for Apache Wayang.
-We started with the Java Platform (part 2) and the Apache Spark implementation follows (W.I.P.) in part three.</p>
-<p>The use case behind this work is an imaginary data collaboration scenario.
-We see this example and the demand for a solution already in many places.
-For us this is motivation enough to propose a solution.
-This would also allow us to do more local data processing, and businesses can stop moving data around the world, but rather care about data locality while they expose and share specific information to others by using data federation.
-This reduces complexity of data management and cost dramatically.</p>
-<p>For this purpose, we illustrate a cross organizational data sharing scenario from the finance sector soon.
-This analysis pattern will also be relevant in the context of data analysis along supply chains, another typical example where data from many stakeholder together is needed but never managed in one place, for good reasons.</p>
-<p>Data federation can help us to unlock the hidden value of all those isolated data lakes.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="a-cross-organizational-data-sharing-scenario">A cross organizational data sharing scenario<a href="#a-cross-organizational-data-sharing-scenario" class="hash-link" aria-label="Direct link to A cross organizational data sharing scenario" title="Direct link to A cross organizational data sharing scenario"></a></h2>
-<p>Our goal is the implementation of a cross organization decentralized data processing scenario, in which protected local data should be processed in combination with public data from public sources in a collaborative manner.
-Instead of copying all data into a central data lake or a central data platform we decided to use federated analytics.
-Apache Wayang is the tool we work with.
-In our case, the public data is hosted on publicly available websites or data pods.
-A client can use the HTTP(S) protocol to read the data which is given in a well defined format.
-For simplicity we decided to use CSV format.
-When we look into the data of each participant we have a different perspective.</p>
-<p>Our processing procedure should calculate a particular metric on the <em>local data</em> of each participant.
-An example of such a metric is the average spending of all users on a particular product category per month.
-This can vary from partner to partner, hence, we want to be able to calculate a peer-group comparison so that each partner can see its own metric compared with a global average calculated from contributions by all partners.
-Such a process requires global averaging and local averaging.
-And due to governance constraints, we can’t bring all raw data together in one place.</p>
-<p>Instead, we want to use Apache Wayang for this purpose.
-We simplify the procedure and split it into two phases.
-Phase one is the process, which allows each participant to calculate the local metrics.
-This requires only local data. The second phase requires data from all collaborating partners.
-The monthly sum and counter values per partner and category are needed in one place by all other parties.
-Hence, the algorithm of the first phase stores the local results locally, and the contributions to the global results in an externally accessible Kafka topic.
-We assume this is done by each of the partners.</p>
-<p>Now we have a scenario, in which an Apache Wayang process must be able to read data from multiple Apache Kafka topics from multiple Apache Kafka clusters but finally writes into a single Kafka topic, which then can be accessed by all the participating clients.</p>
-<p><img decoding="async" loading="lazy" alt="images/image-1.png" src="/assets/images/image-1-9cc35d5aea2b867d7e5759a96bd02334.png" width="904" height="550" class="img_ev3q"></p>
-<p>The illustration shows the data flows in such a scenario.
-Jobs with red border are executed by the participants in isolation within their own data processing environments.
-But they share some of the data, using publicly accessible Kafka topics, marked by A. Job 4 is the Apache Wayang job in our focus: here we intent to read data from 3 different source systems, and write results into a fourth system (marked as B), which can be accesses by all participants again.</p>
-<p>With this in mind we want to implement an Apache Wayang application which implements the illustrated <em>Job 4</em>.
-Since as of today, there is now <em>KafkaSource</em> and <em>KafkaSink</em> available in Apache Wayang, an implementation of both will be our first step.
-Our assumption is, that in the beginning, there won’t be much data.</p>
-<p>Apache Spark is not required to cope with the load, but we expect, that in the future, a single Java application would not be able to handle our workload.
-Hence, we want to utilize the Apache Wayang abstraction over multiple processing platforms, starting with Java.
-Later, we want to switch to Apache Spark.</p></div><footer class="row docusaurus-mt-lg"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/kafka">kafka</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/cross-organization-data-collaboration">cross organization [...]
+<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
+<p>The ASF provides a robust infrastructure for open communities of software developers. We can share ideas, combine forces, contribute code, docs, review-energy, art work, and from time to time we can nail it down. A release defines an intermediate result of the continuous community work.</p>
+<p>How we do such a release in the Apache Wayang team is an essential aspect towards graduation. First of all, there are some references to take into account, such as:</p>
+<ul>
+<li><a href="https://www.apache.org/legal/release-policy.html" target="_blank" rel="noopener noreferrer">https://www.apache.org/legal/release-policy.html</a></li>
+<li><a href="https://maven.apache.org/maven-release/maven-release-plugin/" target="_blank" rel="noopener noreferrer">https://maven.apache.org/maven-release/maven-release-plugin/</a></li>
+</ul>
+<p>Assuming you are (P)PMC, and assuming that you have the right permissions for such a release, you can follow the path as described in this guide:</p>
+<ul>
+<li><a href="https://plc4x.apache.org/developers/release/release.html" target="_blank" rel="noopener noreferrer">https://plc4x.apache.org/developers/release/release.html</a></li>
+</ul>
+<p>I tried to follow exactly this procedure, several times. I failed. Here I share the current status of my <strong>release attempts</strong>.</p>
+<p>I plan a longer tour, and do not want to block the project for a long time.
+Hence I create this draft, and I hope we can unblock this project as soon as possible.</p>
+<h2 class="anchor anchorWithStickyNavbar_LWe7" id="status">Status:<a href="#status" class="hash-link" aria-label="Direct link to Status:" title="Direct link to Status:"></a></h2>
+<p>I am not able to conduct the <em>mvn release:perform</em> step.
+Anything before worked, sometimes only after some digging, but it worked.</p>
+<ul>
+<li>
+<p>We assume, that due to my membership in two ASF incubator projects I am not able to upload the artefacts to the Nexus repository (H1).</p>
+</li>
+<li>
+<p>It can be, that I have not the correct user and password in my <em>settings.xml</em> file (H2).</p>
+</li>
+</ul>
+<p>But I tested a manual login to the nexus server <a href="https://repository.apache.org/service/local/staging/deploy/maven2" target="_blank" rel="noopener noreferrer">https://repository.apache.org/service/local/staging/deploy/maven2</a> with success. And beyond that I have no idea how I can verify this detail alone.</p>
+<h2 class="anchor anchorWithStickyNavbar_LWe7" id="idea--proposal">Idea / Proposal<a href="#idea--proposal" class="hash-link" aria-label="Direct link to Idea / Proposal" title="Direct link to Idea / Proposal"></a></h2>
+<p>(1) It would be great, if someone - who has done a release in any other ASF project or in Apache Wayang - could follow the steps I share, so that we can check where the problem hides itself.</p>
+<p>(2) As a follow-up task, I suggest to add a <strong>Release Guide</strong> to the Apache Wayang project, including release manager onboarding steps, and checklists for the particular project, derived from the referenced sources which are listed above.</p>
+<p>But for now it is all about sharing the status (as I did serveral times on multiple chanels, including JIRA, Slack, Mailing lists) and finding a solution for Apache Wayang release 1.0.</p>
+<h2 class="anchor anchorWithStickyNavbar_LWe7" id="latest-error">Latest Error:<a href="#latest-error" class="hash-link" aria-label="Direct link to Latest Error:" title="Direct link to Latest Error:"></a></h2>
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn release:perform -X -DskipTests</span><br></span></code></pre><div class="buttonGroup__atx"><button ty [...]
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">[INFO] Caused by: org.eclipse.aether.deployment.DeploymentException: Failed to deploy artifacts: Could no [...]
+<h2 class="anchor anchorWithStickyNavbar_LWe7" id="activity-log">Activity Log<a href="#activity-log" class="hash-link" aria-label="Direct link to Activity Log" title="Direct link to Activity Log"></a></h2>
+<div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn release:clean</span><br></span><span class="token-line" style="color:#393A34"><span cla [...]
+<div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn release:prepare </span><span class="token parameter variable" style="color:#36acaa">-Da [...]
+<div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token punctuation" style="color:#393A34">[</span><span class="token plain">ERROR</span><span class="toke [...]
+<div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn versions:set </span><span class="token parameter variable" style="color:#36acaa">-DnewV [...]
+<div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn release:prepare </span><span class="token parameter variable" style="color:#36acaa">-Da [...]
+<div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token punctuation" style="color:#393A34">[</span><span class="token plain">ERROR</span><span class="toke [...]
+<div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token function" style="color:#d73a49">git</span><span class="token plain"> status</span><br></span><span [...]
+<div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn release:prepare </span><span class="token parameter variable" style="color:#36acaa">-Da [...]
+<div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">Caused by: org.eclipse.aether.transfer.NoRepositoryConnectorException: Blocked mirror </spa [...]
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="dependency-on-jdk-11-during-release">Dependency on JDK-11 during release<a href="#dependency-on-jdk-11-during-release" class="hash-link" aria-label="Direct link to Dependency on JDK-11 during release" title="Direct link to Dependency on JDK-11 during release"></a></h3>
+<blockquote>
+<p>FIXED with local JDK11 Setup</p>
+</blockquote>
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">brew install openjdk@11</span><br></span><span class="token-line" style="color:#393A34"><span class="toke [...]
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="manual-update-of-release-version">Manual update of release-version<a href="#manual-update-of-release-version" class="hash-link" aria-label="Direct link to Manual update of release-version" title="Direct link to Manual update of release-version"></a></h3>
+<p>During the release procedure, do I have to set the version here in this configuration section manually?</p>
+<div class="language-xml codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-xml codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token tag punctuation" style="color:#393A34"><</span><span class="token tag" style="color:#00009f">plug [...]
+<blockquote>
+<p>It seems that these properties must be updated manually.</p>
+</blockquote>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="warning-regarding-illegal-reflective-access-operation">Warning regarding "illegal reflective access operation"<a href="#warning-regarding-illegal-reflective-access-operation" class="hash-link" aria-label="Direct link to Warning regarding "illegal reflective access operation"" title="Direct link to Warning regarding "illegal reflective access operation""></a></h3>
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">[ERROR] WARNING: An illegal reflective access operation has occurred</span><br></span><span class="token- [...]
+<blockquote>
+<p><strong>This is still an OPEN ISSUE !</strong></p>
+</blockquote>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="rat-check-fails">RAT Check fails<a href="#rat-check-fails" class="hash-link" aria-label="Direct link to RAT Check fails" title="Direct link to RAT Check fails"></a></h3>
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">[INFO] [ERROR] Failed to execute goal org.apache.rat:apache-rat-plugin:0.13:check (license-check) on proj [...]
+<div class="language-cat codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-cat codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#393A34" [...]
+<blockquote>
+<p><em>FIXED by adding .java-versions to .gitignore</em></p>
+</blockquote>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="tag-could-not-be-created-in-scm">Tag could not be created in SCM.<a href="#tag-could-not-be-created-in-scm" class="hash-link" aria-label="Direct link to Tag could not be created in SCM." title="Direct link to Tag could not be created in SCM."></a></h3>
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">[ERROR] Failed to execute goal org.apache.maven.plugins:maven-release-plugin:3.0.1:prepare (default-cli) [...]
+<blockquote>
+<p>FIXED by manual changes in pom.xml.</p>
+</blockquote>
+<div class="language-xml codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-xml codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#393A34" [...]
+<div class="language-xml codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-xml codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token tag punctuation" style="color:#393A34"><</span><span class="token tag" style="color:#00009f">conf [...]
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn release:prepare -Darguments='-DskipTests=True -Dresume=True'</span><br></span></code></pre> [...]
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn clean package</span><br></span><span class="token-line" style="color:#393A34"><span class="token plai [...]
+<blockquote>
+<p>So far so good. But now the sun went down.</p>
+</blockquote>
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">[INFO] [ERROR] Failed to execute goal org.apache.maven.plugins:maven-deploy-plugin:3.0.0-M1:deploy (defau [...]
+<p>The null pointer exception indicates problems in <em>settings.xml</em>. This could be fixed.
+But now, the error changes with and I am still not able to stage the build results.</p>
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn release:perform -X -DskipTests</span><br></span></code></pre><div class="buttonGroup__atx"><button ty [...]
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">[INFO] Caused by: org.eclipse.aether.deployment.DeploymentException: Failed to deploy artifacts: Could no [...]
+<h2 class="anchor anchorWithStickyNavbar_LWe7" id="checklist-for-next-iteration">Checklist for next iteration:<a href="#checklist-for-next-iteration" class="hash-link" aria-label="Direct link to Checklist for next iteration:" title="Direct link to Checklist for next iteration:"></a></h2>
+<p>The error you're encountering indicates that there is a problem with deploying the artifacts using the <code>maven-deploy-plugin</code>. The root cause of the error is a <code>NullPointerException</code> during the deployment process, specifically related to the <code>org.eclipse.aether.transfer.ArtifactTransferException</code>.</p>
+<p>Here's a step-by-step guide to troubleshoot and resolve this issue:</p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="1-check-maven-settings">1. Check Maven Settings<a href="#1-check-maven-settings" class="hash-link" aria-label="Direct link to 1. Check Maven Settings" title="Direct link to 1. Check Maven Settings"></a></h3>
+<p>Ensure that your Maven settings (<code>settings.xml</code>) are correctly configured for deployment. Verify that the repository settings and credentials are correctly specified.</p>
+<p><strong>[DONE]</strong></p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="2-verify-repository-url">2. Verify Repository URL<a href="#2-verify-repository-url" class="hash-link" aria-label="Direct link to 2. Verify Repository URL" title="Direct link to 2. Verify Repository URL"></a></h3>
+<p>Make sure the repository URL in your <code>pom.xml</code> or <code>settings.xml</code> is correct and reachable. The URL should point to the correct staging repository for deployment.</p>
+<p><em>[OPEN]</em> - I did not touch it, so I guess it is correct in the <em>pom.xml</em>.</p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="3-maven-version-compatibility">3. Maven Version Compatibility<a href="#3-maven-version-compatibility" class="hash-link" aria-label="Direct link to 3. Maven Version Compatibility" title="Direct link to 3. Maven Version Compatibility"></a></h3>
+<p>Verify that you are using a compatible version of Maven. Sometimes, upgrading or downgrading Maven can resolve such issues.</p>
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">Apache Maven 3.9.6 (bc0240f3c744dd6b6ec2920b3cd08dcc295161ae)</span><br></span><span class="token-line" s [...]
+<p><strong>[DONE]</strong></p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="4-check-for-network-issues">4. Check for Network Issues<a href="#4-check-for-network-issues" class="hash-link" aria-label="Direct link to 4. Check for Network Issues" title="Direct link to 4. Check for Network Issues"></a></h3>
+<p>Ensure there are no network issues that might be causing problems in connecting to the repository. Sometimes, network configurations, firewalls, or proxy settings can interfere with the deployment process.</p>
+<p><strong>[DONE]</strong></p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="5-update-maven-plugins">5. Update Maven Plugins<a href="#5-update-maven-plugins" class="hash-link" aria-label="Direct link to 5. Update Maven Plugins" title="Direct link to 5. Update Maven Plugins"></a></h3>
+<p>Ensure that you are using the latest versions of the Maven plugins. Sometimes, bugs in older versions can cause unexpected issues.</p>
+<p><em>[OPEN]</em></p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="6-configure-the-maven-deploy-plugin-in-pomxml">6. Configure the <code>maven-deploy-plugin</code> in <code>pom.xml</code><a href="#6-configure-the-maven-deploy-plugin-in-pomxml" class="hash-link" aria-label="Direct link to 6-configure-the-maven-deploy-plugin-in-pomxml" title="Direct link to 6-configure-the-maven-deploy-plugin-in-pomxml"></a></h3>
+<p>Make sure the <code>maven-deploy-plugin</code> is correctly configured in your <code>pom.xml</code>. Here's an example configuration:</p>
+<div class="language-xml codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-xml codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token tag punctuation" style="color:#393A34"><</span><span class="token tag" style="color:#00009f">plug [...]
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="7-increase-verbose-logging">7. Increase Verbose Logging<a href="#7-increase-verbose-logging" class="hash-link" aria-label="Direct link to 7. Increase Verbose Logging" title="Direct link to 7. Increase Verbose Logging"></a></h3>
+<p>Enable verbose logging to get more details about the error. You can do this by adding the <code>-X</code> flag when running the Maven command:</p>
+<div class="language-sh codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-sh codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn clean deploy </span><span class="token parameter variable" style="color:#36acaa">-X</span>< [...]
+<p><strong>[DONE]</strong></p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="8-retry-with-a-clean-local-repository">8. Retry with a Clean Local Repository<a href="#8-retry-with-a-clean-local-repository" class="hash-link" aria-label="Direct link to 8. Retry with a Clean Local Repository" title="Direct link to 8. Retry with a Clean Local Repository"></a></h3>
+<p>Sometimes, a corrupt local repository can cause issues. Try cleaning your local Maven repository and re-running the deployment:</p>
+<div class="language-sh codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-sh codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn clean </span><span class="token function" style="color:#d73a49">install</span><span class=" [...]
+<p><em>[OPEN]</em> - Does not match to an authentication issue.</p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="9-check-for-missing-credentials">9. Check for Missing Credentials<a href="#9-check-for-missing-credentials" class="hash-link" aria-label="Direct link to 9. Check for Missing Credentials" title="Direct link to 9. Check for Missing Credentials"></a></h3>
+<p>Ensure that the credentials for the repository are correctly set up in your <code>settings.xml</code>:</p>
+<div class="language-xml codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-xml codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token tag punctuation" style="color:#393A34"><</span><span class="token tag" style="color:#00009f">serv [...]
+<p><strong>[DONE]</strong></p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="10-review-the-full-stack-trace">10. Review the Full Stack Trace<a href="#10-review-the-full-stack-trace" class="hash-link" aria-label="Direct link to 10. Review the Full Stack Trace" title="Direct link to 10. Review the Full Stack Trace"></a></h3>
+<p>The full stack trace indicates a <code>NullPointerException</code>:</p>
+<div class="language-plaintext codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-plaintext codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">Caused by: java.lang.NullPointerException</span><br></span><span class="token-lin [...]
+<p>This suggests that there might be an issue with how credentials are being handled. Double-check that the credentials are being correctly passed and processed.</p>
+<p><strong>[DONE]</strong></p></div><footer class="row docusaurus-mt-lg"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/asf">ASF</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/release">release</a></li></ul></div></footer></article><article class="margi [...]
Thanks to the extensible nature of Wayang, the only core changes we had to do were introducing the concept of a <code>Model</code> and implement a new driver for the newly added platform.</p>
<h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-1-introducing-a-model">Step 1: Introducing a Model<a href="#step-1-introducing-a-model" class="hash-link" aria-label="Direct link to Step 1: Introducing a Model" title="Direct link to Step 1: Introducing a Model"></a></h2>
<p>With respect to the model, we followed Wayang’s abstraction philosophy: We created a <code>Model</code> interface to be used as input or output by Wayang operators and then extended it for the platform-specific operators. Different model interfaces can be found here:</p>
@@ -346,7 +266,7 @@ be executed on iterables in their respective language.
After that, the API can be added as a module in Wayang, so that
operators will be wrapped and UDFs can be executed in the desired
programming language.</li>
-</ul></div><footer class="row docusaurus-mt-lg"><div class="col col--9"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/python">python</a></li></ul></div><div class="col text--right col--3"><a aria-label="Read more about Pywayang - Apache Wayang's Python API" href="/blog/wayang-python-api"><b>Read M [...]
+</ul></div><footer class="row docusaurus-mt-lg"><div class="col col--9"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/python">python</a></li></ul></div><div class="col text--right col--3"><a aria-label="Read more about Pywayang - Apache Wayang's Python API" href="/blog/wayang-python-api"><b>Read M [...]
Motivated by the learnings from last time, I stated implementing a Kafka Source component and a Kafka Sink component for the Apache Spark platform in Apache Wayang.
In our previous article we shared the results of the work on the frist Apache Kafka integration using the Java Platform.</p>
<p>Let's see how it goes this time with Apache Spark.</p>
@@ -414,7 +334,7 @@ More testing, more serialization schemes, and Kafka Schema Registry support shou
<h2 class="anchor anchorWithStickyNavbar_LWe7" id="outlook">Outlook<a href="#outlook" class="hash-link" aria-label="Direct link to Outlook" title="Direct link to Outlook"></a></h2>
<p>The next part of the article series will cover the real world example as described in image 1.
We will show how analysts and developers can use the Apache Kafka integration for Apache Wayang to solve cross organizational collaboration issues.
-Therefore, we will bring all puzzles together, and show the full implementation of the multi organizational data collaboration use case.</p></div><footer class="row docusaurus-mt-lg"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/kafka">kafka</a></li><li class="tag_QGVx"><a class="tag_z [...]
+Therefore, we will bring all puzzles together, and show the full implementation of the multi organizational data collaboration use case.</p></div><footer class="row docusaurus-mt-lg"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/kafka">kafka</a></li><li class="tag_QGVx"><a class="tag_z [...]
We look into the “Read- and Write-Path” for our data items, called <em>DataQuanta</em>.</p>
<h2 class="anchor anchorWithStickyNavbar_LWe7" id="apache-wayangs-read--write-path-for-kafka-topics">Apache Wayang’s Read & Write Path for Kafka topics<a href="#apache-wayangs-read--write-path-for-kafka-topics" class="hash-link" aria-label="Direct link to Apache Wayang’s Read & Write Path for Kafka topics" title="Direct link to Apache Wayang’s Read & Write Path for Kafka topics"></a></h2>
<p>To describe the read and write paths for data in the context of the created Apache Wayang code snippet, the primary classes and interfaces we need to understand are as follows:</p>
@@ -471,7 +391,50 @@ The layer above handles the mapping of the components at startup time.
All this wiring is needed to keep Wayang open and flexible so that multiple external systems can be used in a variety of combinations and using multiple target platforms in combinations.</p>
<h2 class="anchor anchorWithStickyNavbar_LWe7" id="outlook">Outlook<a href="#outlook" class="hash-link" aria-label="Direct link to Outlook" title="Direct link to Outlook"></a></h2>
<p>The next part of the article series will cover the creation of an Kafka Source and Sink component for the Apache Spark platform, which allows our use case to scale.
-Finally, in part four we bring all puzzles together, and show the full implementation of the multi organizational data collaboration use case.</p></div><footer class="row docusaurus-mt-lg"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/kafka">kafka</a></li><li class="tag_QGVx"><a class= [...]
+Finally, in part four we bring all puzzles together, and show the full implementation of the multi organizational data collaboration use case.</p></div><footer class="row docusaurus-mt-lg"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/kafka">kafka</a></li><li class="tag_QGVx"><a class= [...]
+<p>This article is the first of a four part series about federated data analysis using Apache Wayang.
+The first article starts with an introduction of a typical data colaboration scenario which will emerge in our digital future.</p>
+<p>In part two and three we will share a summary of our Apache Kafka client implementation for Apache Wayang.
+We started with the Java Platform (part 2) and the Apache Spark implementation follows (W.I.P.) in part three.</p>
+<p>The use case behind this work is an imaginary data collaboration scenario.
+We see this example and the demand for a solution already in many places.
+For us this is motivation enough to propose a solution.
+This would also allow us to do more local data processing, and businesses can stop moving data around the world, but rather care about data locality while they expose and share specific information to others by using data federation.
+This reduces complexity of data management and cost dramatically.</p>
+<p>For this purpose, we illustrate a cross organizational data sharing scenario from the finance sector soon.
+This analysis pattern will also be relevant in the context of data analysis along supply chains, another typical example where data from many stakeholder together is needed but never managed in one place, for good reasons.</p>
+<p>Data federation can help us to unlock the hidden value of all those isolated data lakes.</p>
+<h2 class="anchor anchorWithStickyNavbar_LWe7" id="a-cross-organizational-data-sharing-scenario">A cross organizational data sharing scenario<a href="#a-cross-organizational-data-sharing-scenario" class="hash-link" aria-label="Direct link to A cross organizational data sharing scenario" title="Direct link to A cross organizational data sharing scenario"></a></h2>
+<p>Our goal is the implementation of a cross organization decentralized data processing scenario, in which protected local data should be processed in combination with public data from public sources in a collaborative manner.
+Instead of copying all data into a central data lake or a central data platform we decided to use federated analytics.
+Apache Wayang is the tool we work with.
+In our case, the public data is hosted on publicly available websites or data pods.
+A client can use the HTTP(S) protocol to read the data which is given in a well defined format.
+For simplicity we decided to use CSV format.
+When we look into the data of each participant we have a different perspective.</p>
+<p>Our processing procedure should calculate a particular metric on the <em>local data</em> of each participant.
+An example of such a metric is the average spending of all users on a particular product category per month.
+This can vary from partner to partner, hence, we want to be able to calculate a peer-group comparison so that each partner can see its own metric compared with a global average calculated from contributions by all partners.
+Such a process requires global averaging and local averaging.
+And due to governance constraints, we can’t bring all raw data together in one place.</p>
+<p>Instead, we want to use Apache Wayang for this purpose.
+We simplify the procedure and split it into two phases.
+Phase one is the process, which allows each participant to calculate the local metrics.
+This requires only local data. The second phase requires data from all collaborating partners.
+The monthly sum and counter values per partner and category are needed in one place by all other parties.
+Hence, the algorithm of the first phase stores the local results locally, and the contributions to the global results in an externally accessible Kafka topic.
+We assume this is done by each of the partners.</p>
+<p>Now we have a scenario, in which an Apache Wayang process must be able to read data from multiple Apache Kafka topics from multiple Apache Kafka clusters but finally writes into a single Kafka topic, which then can be accessed by all the participating clients.</p>
+<p><img decoding="async" loading="lazy" alt="images/image-1.png" src="/assets/images/image-1-9cc35d5aea2b867d7e5759a96bd02334.png" width="904" height="550" class="img_ev3q"></p>
+<p>The illustration shows the data flows in such a scenario.
+Jobs with red border are executed by the participants in isolation within their own data processing environments.
+But they share some of the data, using publicly accessible Kafka topics, marked by A. Job 4 is the Apache Wayang job in our focus: here we intent to read data from 3 different source systems, and write results into a fourth system (marked as B), which can be accesses by all participants again.</p>
+<p>With this in mind we want to implement an Apache Wayang application which implements the illustrated <em>Job 4</em>.
+Since as of today, there is now <em>KafkaSource</em> and <em>KafkaSink</em> available in Apache Wayang, an implementation of both will be our first step.
+Our assumption is, that in the beginning, there won’t be much data.</p>
+<p>Apache Spark is not required to cope with the load, but we expect, that in the future, a single Java application would not be able to handle our workload.
+Hence, we want to utilize the Apache Wayang abstraction over multiple processing platforms, starting with Java.
+Later, we want to switch to Apache Spark.</p></div><footer class="row docusaurus-mt-lg"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/kafka">kafka</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/cross-organization-data-collaboration">cross organization [...]
<p> Apache Wayang is an effort undergoing incubation at The Apache Software Foundation (ASF), sponsored by the Apache Incubator. Incubation is required of all newly accepted projects until a further review indicates that the infrastructure, communications, and decision making process have stabilized in a manner consistent with other successful ASF projects. While incubation status is not necessarily a reflection of the completeness or stability of the code, it does indicate that th [...]
<p>
Copyright © 2024 The Apache Software Foundation, Licensed under the Apache License, Version 2.0. <br>
diff --git a/blog/tags/wayang/page/2/index.html b/blog/tags/wayang/page/2/index.html
deleted file mode 100644
index 6443f7c1..00000000
--- a/blog/tags/wayang/page/2/index.html
+++ /dev/null
@@ -1,63 +0,0 @@
-<!doctype html>
-<html lang="en" dir="ltr" class="blog-wrapper blog-tags-post-list-page plugin-blog plugin-id-default" data-has-hydrated="false">
-<head>
-<meta charset="UTF-8">
-<meta name="generator" content="Docusaurus v3.1.1">
-<title data-rh="true">12 posts tagged with "wayang" | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/blog/tags/wayang/page/2"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="d [...]
-<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.01bb8940.js" defer="defer"></script>
-<script src="/assets/js/main.0cc2e93a.js" defer="defer"></script>
-</head>
-<body class="navigation-with-keyboard">
-<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
-<p>This article is the first of a four part series about federated data analysis using Apache Wayang.
-The first article starts with an introduction of a typical data colaboration scenario which will emerge in our digital future.</p>
-<p>In part two and three we will share a summary of our Apache Kafka client implementation for Apache Wayang.
-We started with the Java Platform (part 2) and the Apache Spark implementation follows (W.I.P.) in part three.</p>
-<p>The use case behind this work is an imaginary data collaboration scenario.
-We see this example and the demand for a solution already in many places.
-For us this is motivation enough to propose a solution.
-This would also allow us to do more local data processing, and businesses can stop moving data around the world, but rather care about data locality while they expose and share specific information to others by using data federation.
-This reduces complexity of data management and cost dramatically.</p>
-<p>For this purpose, we illustrate a cross organizational data sharing scenario from the finance sector soon.
-This analysis pattern will also be relevant in the context of data analysis along supply chains, another typical example where data from many stakeholder together is needed but never managed in one place, for good reasons.</p>
-<p>Data federation can help us to unlock the hidden value of all those isolated data lakes.</p>
-<h2 class="anchor anchorWithStickyNavbar_LWe7" id="a-cross-organizational-data-sharing-scenario">A cross organizational data sharing scenario<a href="#a-cross-organizational-data-sharing-scenario" class="hash-link" aria-label="Direct link to A cross organizational data sharing scenario" title="Direct link to A cross organizational data sharing scenario"></a></h2>
-<p>Our goal is the implementation of a cross organization decentralized data processing scenario, in which protected local data should be processed in combination with public data from public sources in a collaborative manner.
-Instead of copying all data into a central data lake or a central data platform we decided to use federated analytics.
-Apache Wayang is the tool we work with.
-In our case, the public data is hosted on publicly available websites or data pods.
-A client can use the HTTP(S) protocol to read the data which is given in a well defined format.
-For simplicity we decided to use CSV format.
-When we look into the data of each participant we have a different perspective.</p>
-<p>Our processing procedure should calculate a particular metric on the <em>local data</em> of each participant.
-An example of such a metric is the average spending of all users on a particular product category per month.
-This can vary from partner to partner, hence, we want to be able to calculate a peer-group comparison so that each partner can see its own metric compared with a global average calculated from contributions by all partners.
-Such a process requires global averaging and local averaging.
-And due to governance constraints, we can’t bring all raw data together in one place.</p>
-<p>Instead, we want to use Apache Wayang for this purpose.
-We simplify the procedure and split it into two phases.
-Phase one is the process, which allows each participant to calculate the local metrics.
-This requires only local data. The second phase requires data from all collaborating partners.
-The monthly sum and counter values per partner and category are needed in one place by all other parties.
-Hence, the algorithm of the first phase stores the local results locally, and the contributions to the global results in an externally accessible Kafka topic.
-We assume this is done by each of the partners.</p>
-<p>Now we have a scenario, in which an Apache Wayang process must be able to read data from multiple Apache Kafka topics from multiple Apache Kafka clusters but finally writes into a single Kafka topic, which then can be accessed by all the participating clients.</p>
-<p><img decoding="async" loading="lazy" alt="images/image-1.png" src="/assets/images/image-1-9cc35d5aea2b867d7e5759a96bd02334.png" width="904" height="550" class="img_ev3q"></p>
-<p>The illustration shows the data flows in such a scenario.
-Jobs with red border are executed by the participants in isolation within their own data processing environments.
-But they share some of the data, using publicly accessible Kafka topics, marked by A. Job 4 is the Apache Wayang job in our focus: here we intent to read data from 3 different source systems, and write results into a fourth system (marked as B), which can be accesses by all participants again.</p>
-<p>With this in mind we want to implement an Apache Wayang application which implements the illustrated <em>Job 4</em>.
-Since as of today, there is now <em>KafkaSource</em> and <em>KafkaSink</em> available in Apache Wayang, an implementation of both will be our first step.
-Our assumption is, that in the beginning, there won’t be much data.</p>
-<p>Apache Spark is not required to cope with the load, but we expect, that in the future, a single Java application would not be able to handle our workload.
-Hence, we want to utilize the Apache Wayang abstraction over multiple processing platforms, starting with Java.
-Later, we want to switch to Apache Spark.</p></div><footer class="row docusaurus-mt-lg"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/kafka">kafka</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/cross-organization-data-collaboration">cross organization [...]
- <p> Apache Wayang is an effort undergoing incubation at The Apache Software Foundation (ASF), sponsored by the Apache Incubator. Incubation is required of all newly accepted projects until a further review indicates that the infrastructure, communications, and decision making process have stabilized in a manner consistent with other successful ASF projects. While incubation status is not necessarily a reflection of the completeness or stability of the code, it does indicate that th [...]
- <p>
- Copyright © 2024 The Apache Software Foundation, Licensed under the Apache License, Version 2.0. <br>
- Apache, the names of Apache projects, and the feather logo are either registered trademarks or trademarks of the Apache Software Foundation in the United States and/or other countries.
- </p>
- </div></div></div></div></footer></div>
-</body>
-</html>
\ No newline at end of file
diff --git a/blog/wayang-federated-ai/index.html b/blog/wayang-federated-ai/index.html
index e2f3259c..91d14a8f 100644
--- a/blog/wayang-federated-ai/index.html
+++ b/blog/wayang-federated-ai/index.html
@@ -5,11 +5,11 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">Wayang and the Federated AI | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/blog/wayang-federated-ai"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docusaurus_t [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.01bb8940.js" defer="defer"></script>
-<script src="/assets/js/main.0cc2e93a.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.8489d39d.js" defer="defer"></script>
+<script src="/assets/js/main.4f5176d9.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
-<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
+<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
detecting early signs of cancer cells. Building an AI requires a lot of data and training to achieve
the desired results, and federated learning is an approach to make AI training more viable.
Federated learning (or collaborative learning) is a technique that trains AI models on data
diff --git a/blog/wayang-python-api/index.html b/blog/wayang-python-api/index.html
index fca08eae..e9dac1ff 100644
--- a/blog/wayang-python-api/index.html
+++ b/blog/wayang-python-api/index.html
@@ -5,11 +5,11 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">Pywayang - Apache Wayang's Python API | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/blog/wayang-python-api"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name= [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.01bb8940.js" defer="defer"></script>
-<script src="/assets/js/main.0cc2e93a.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.8489d39d.js" defer="defer"></script>
+<script src="/assets/js/main.4f5176d9.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
-<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
+<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
important. However, navigating through a multitude of tools and
languages often is a major inconvenience.
Apache Wayang's upcoming Python API will allow you to seamlessly
@@ -92,7 +92,7 @@ to the Python worker will have to be made for a given pipeline.</p>
<p>As the Python API is currently in development and we are applying
finishing touches, this article serves as an outlook for what users can
expect to see soon.</p>
-<p>Author: <a href="https://github.com/juripetersen" target="_blank" rel="noopener noreferrer">juripetersen</a></p></div><footer class="row docusaurus-mt-lg blogPostFooterDetailsFull_mRVl"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/python">python</a></li></ul></div></footer></articl [...]
+<p>Author: <a href="https://github.com/juripetersen" target="_blank" rel="noopener noreferrer">juripetersen</a></p></div><footer class="row docusaurus-mt-lg blogPostFooterDetailsFull_mRVl"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/python">python</a></li></ul></div></footer></articl [...]
<p> Apache Wayang is an effort undergoing incubation at The Apache Software Foundation (ASF), sponsored by the Apache Incubator. Incubation is required of all newly accepted projects until a further review indicates that the infrastructure, communications, and decision making process have stabilized in a manner consistent with other successful ASF projects. While incubation status is not necessarily a reflection of the completeness or stability of the code, it does indicate that th [...]
<p>
Copyright © 2024 The Apache Software Foundation, Licensed under the Apache License, Version 2.0. <br>
diff --git a/blog/wayang-release-odysse/index.html b/blog/wayang-release-odysse/index.html
new file mode 100644
index 00000000..d5899d23
--- /dev/null
+++ b/blog/wayang-release-odysse/index.html
@@ -0,0 +1,140 @@
+<!doctype html>
+<html lang="en" dir="ltr" class="blog-wrapper blog-post-page plugin-blog plugin-id-default" data-has-hydrated="false">
+<head>
+<meta charset="UTF-8">
+<meta name="generator" content="Docusaurus v3.1.1">
+<title data-rh="true">Apache Wayang Release Odysse | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/blog/wayang-release-odysse"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docusauru [...]
+<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
+<script src="/assets/js/runtime~main.8489d39d.js" defer="defer"></script>
+<script src="/assets/js/main.4f5176d9.js" defer="defer"></script>
+</head>
+<body class="navigation-with-keyboard">
+<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
+<p>The ASF provides a robust infrastructure for open communities of software developers. We can share ideas, combine forces, contribute code, docs, review-energy, art work, and from time to time we can nail it down. A release defines an intermediate result of the continuous community work.</p>
+<p>How we do such a release in the Apache Wayang team is an essential aspect towards graduation. First of all, there are some references to take into account, such as:</p>
+<ul>
+<li><a href="https://www.apache.org/legal/release-policy.html" target="_blank" rel="noopener noreferrer">https://www.apache.org/legal/release-policy.html</a></li>
+<li><a href="https://maven.apache.org/maven-release/maven-release-plugin/" target="_blank" rel="noopener noreferrer">https://maven.apache.org/maven-release/maven-release-plugin/</a></li>
+</ul>
+<p>Assuming you are (P)PMC, and assuming that you have the right permissions for such a release, you can follow the path as described in this guide:</p>
+<ul>
+<li><a href="https://plc4x.apache.org/developers/release/release.html" target="_blank" rel="noopener noreferrer">https://plc4x.apache.org/developers/release/release.html</a></li>
+</ul>
+<p>I tried to follow exactly this procedure, several times. I failed. Here I share the current status of my <strong>release attempts</strong>.</p>
+<p>I plan a longer tour, and do not want to block the project for a long time.
+Hence I create this draft, and I hope we can unblock this project as soon as possible.</p>
+<h2 class="anchor anchorWithStickyNavbar_LWe7" id="status">Status:<a href="#status" class="hash-link" aria-label="Direct link to Status:" title="Direct link to Status:"></a></h2>
+<p>I am not able to conduct the <em>mvn release:perform</em> step.
+Anything before worked, sometimes only after some digging, but it worked.</p>
+<ul>
+<li>
+<p>We assume, that due to my membership in two ASF incubator projects I am not able to upload the artefacts to the Nexus repository (H1).</p>
+</li>
+<li>
+<p>It can be, that I have not the correct user and password in my <em>settings.xml</em> file (H2).</p>
+</li>
+</ul>
+<p>But I tested a manual login to the nexus server <a href="https://repository.apache.org/service/local/staging/deploy/maven2" target="_blank" rel="noopener noreferrer">https://repository.apache.org/service/local/staging/deploy/maven2</a> with success. And beyond that I have no idea how I can verify this detail alone.</p>
+<h2 class="anchor anchorWithStickyNavbar_LWe7" id="idea--proposal">Idea / Proposal<a href="#idea--proposal" class="hash-link" aria-label="Direct link to Idea / Proposal" title="Direct link to Idea / Proposal"></a></h2>
+<p>(1) It would be great, if someone - who has done a release in any other ASF project or in Apache Wayang - could follow the steps I share, so that we can check where the problem hides itself.</p>
+<p>(2) As a follow-up task, I suggest to add a <strong>Release Guide</strong> to the Apache Wayang project, including release manager onboarding steps, and checklists for the particular project, derived from the referenced sources which are listed above.</p>
+<p>But for now it is all about sharing the status (as I did serveral times on multiple chanels, including JIRA, Slack, Mailing lists) and finding a solution for Apache Wayang release 1.0.</p>
+<h2 class="anchor anchorWithStickyNavbar_LWe7" id="latest-error">Latest Error:<a href="#latest-error" class="hash-link" aria-label="Direct link to Latest Error:" title="Direct link to Latest Error:"></a></h2>
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn release:perform -X -DskipTests</span><br></span></code></pre><div class="buttonGroup__atx"><button ty [...]
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">[INFO] Caused by: org.eclipse.aether.deployment.DeploymentException: Failed to deploy artifacts: Could no [...]
+<h2 class="anchor anchorWithStickyNavbar_LWe7" id="activity-log">Activity Log<a href="#activity-log" class="hash-link" aria-label="Direct link to Activity Log" title="Direct link to Activity Log"></a></h2>
+<div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn release:clean</span><br></span><span class="token-line" style="color:#393A34"><span cla [...]
+<div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn release:prepare </span><span class="token parameter variable" style="color:#36acaa">-Da [...]
+<div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token punctuation" style="color:#393A34">[</span><span class="token plain">ERROR</span><span class="toke [...]
+<div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn versions:set </span><span class="token parameter variable" style="color:#36acaa">-DnewV [...]
+<div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn release:prepare </span><span class="token parameter variable" style="color:#36acaa">-Da [...]
+<div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token punctuation" style="color:#393A34">[</span><span class="token plain">ERROR</span><span class="toke [...]
+<div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token function" style="color:#d73a49">git</span><span class="token plain"> status</span><br></span><span [...]
+<div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn release:prepare </span><span class="token parameter variable" style="color:#36acaa">-Da [...]
+<div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">Caused by: org.eclipse.aether.transfer.NoRepositoryConnectorException: Blocked mirror </spa [...]
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="dependency-on-jdk-11-during-release">Dependency on JDK-11 during release<a href="#dependency-on-jdk-11-during-release" class="hash-link" aria-label="Direct link to Dependency on JDK-11 during release" title="Direct link to Dependency on JDK-11 during release"></a></h3>
+<blockquote>
+<p>FIXED with local JDK11 Setup</p>
+</blockquote>
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">brew install openjdk@11</span><br></span><span class="token-line" style="color:#393A34"><span class="toke [...]
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="manual-update-of-release-version">Manual update of release-version<a href="#manual-update-of-release-version" class="hash-link" aria-label="Direct link to Manual update of release-version" title="Direct link to Manual update of release-version"></a></h3>
+<p>During the release procedure, do I have to set the version here in this configuration section manually?</p>
+<div class="language-xml codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-xml codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token tag punctuation" style="color:#393A34"><</span><span class="token tag" style="color:#00009f">plug [...]
+<blockquote>
+<p>It seems that these properties must be updated manually.</p>
+</blockquote>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="warning-regarding-illegal-reflective-access-operation">Warning regarding "illegal reflective access operation"<a href="#warning-regarding-illegal-reflective-access-operation" class="hash-link" aria-label="Direct link to Warning regarding "illegal reflective access operation"" title="Direct link to Warning regarding "illegal reflective access operation""></a></h3>
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">[ERROR] WARNING: An illegal reflective access operation has occurred</span><br></span><span class="token- [...]
+<blockquote>
+<p><strong>This is still an OPEN ISSUE !</strong></p>
+</blockquote>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="rat-check-fails">RAT Check fails<a href="#rat-check-fails" class="hash-link" aria-label="Direct link to RAT Check fails" title="Direct link to RAT Check fails"></a></h3>
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">[INFO] [ERROR] Failed to execute goal org.apache.rat:apache-rat-plugin:0.13:check (license-check) on proj [...]
+<div class="language-cat codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-cat codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#393A34" [...]
+<blockquote>
+<p><em>FIXED by adding .java-versions to .gitignore</em></p>
+</blockquote>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="tag-could-not-be-created-in-scm">Tag could not be created in SCM.<a href="#tag-could-not-be-created-in-scm" class="hash-link" aria-label="Direct link to Tag could not be created in SCM." title="Direct link to Tag could not be created in SCM."></a></h3>
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">[ERROR] Failed to execute goal org.apache.maven.plugins:maven-release-plugin:3.0.1:prepare (default-cli) [...]
+<blockquote>
+<p>FIXED by manual changes in pom.xml.</p>
+</blockquote>
+<div class="language-xml codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-xml codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#393A34" [...]
+<div class="language-xml codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-xml codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token tag punctuation" style="color:#393A34"><</span><span class="token tag" style="color:#00009f">conf [...]
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn release:prepare -Darguments='-DskipTests=True -Dresume=True'</span><br></span></code></pre> [...]
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn clean package</span><br></span><span class="token-line" style="color:#393A34"><span class="token plai [...]
+<blockquote>
+<p>So far so good. But now the sun went down.</p>
+</blockquote>
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">[INFO] [ERROR] Failed to execute goal org.apache.maven.plugins:maven-deploy-plugin:3.0.0-M1:deploy (defau [...]
+<p>The null pointer exception indicates problems in <em>settings.xml</em>. This could be fixed.
+But now, the error changes with and I am still not able to stage the build results.</p>
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn release:perform -X -DskipTests</span><br></span></code></pre><div class="buttonGroup__atx"><button ty [...]
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">[INFO] Caused by: org.eclipse.aether.deployment.DeploymentException: Failed to deploy artifacts: Could no [...]
+<h2 class="anchor anchorWithStickyNavbar_LWe7" id="checklist-for-next-iteration">Checklist for next iteration:<a href="#checklist-for-next-iteration" class="hash-link" aria-label="Direct link to Checklist for next iteration:" title="Direct link to Checklist for next iteration:"></a></h2>
+<p>The error you're encountering indicates that there is a problem with deploying the artifacts using the <code>maven-deploy-plugin</code>. The root cause of the error is a <code>NullPointerException</code> during the deployment process, specifically related to the <code>org.eclipse.aether.transfer.ArtifactTransferException</code>.</p>
+<p>Here's a step-by-step guide to troubleshoot and resolve this issue:</p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="1-check-maven-settings">1. Check Maven Settings<a href="#1-check-maven-settings" class="hash-link" aria-label="Direct link to 1. Check Maven Settings" title="Direct link to 1. Check Maven Settings"></a></h3>
+<p>Ensure that your Maven settings (<code>settings.xml</code>) are correctly configured for deployment. Verify that the repository settings and credentials are correctly specified.</p>
+<p><strong>[DONE]</strong></p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="2-verify-repository-url">2. Verify Repository URL<a href="#2-verify-repository-url" class="hash-link" aria-label="Direct link to 2. Verify Repository URL" title="Direct link to 2. Verify Repository URL"></a></h3>
+<p>Make sure the repository URL in your <code>pom.xml</code> or <code>settings.xml</code> is correct and reachable. The URL should point to the correct staging repository for deployment.</p>
+<p><em>[OPEN]</em> - I did not touch it, so I guess it is correct in the <em>pom.xml</em>.</p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="3-maven-version-compatibility">3. Maven Version Compatibility<a href="#3-maven-version-compatibility" class="hash-link" aria-label="Direct link to 3. Maven Version Compatibility" title="Direct link to 3. Maven Version Compatibility"></a></h3>
+<p>Verify that you are using a compatible version of Maven. Sometimes, upgrading or downgrading Maven can resolve such issues.</p>
+<div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">Apache Maven 3.9.6 (bc0240f3c744dd6b6ec2920b3cd08dcc295161ae)</span><br></span><span class="token-line" s [...]
+<p><strong>[DONE]</strong></p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="4-check-for-network-issues">4. Check for Network Issues<a href="#4-check-for-network-issues" class="hash-link" aria-label="Direct link to 4. Check for Network Issues" title="Direct link to 4. Check for Network Issues"></a></h3>
+<p>Ensure there are no network issues that might be causing problems in connecting to the repository. Sometimes, network configurations, firewalls, or proxy settings can interfere with the deployment process.</p>
+<p><strong>[DONE]</strong></p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="5-update-maven-plugins">5. Update Maven Plugins<a href="#5-update-maven-plugins" class="hash-link" aria-label="Direct link to 5. Update Maven Plugins" title="Direct link to 5. Update Maven Plugins"></a></h3>
+<p>Ensure that you are using the latest versions of the Maven plugins. Sometimes, bugs in older versions can cause unexpected issues.</p>
+<p><em>[OPEN]</em></p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="6-configure-the-maven-deploy-plugin-in-pomxml">6. Configure the <code>maven-deploy-plugin</code> in <code>pom.xml</code><a href="#6-configure-the-maven-deploy-plugin-in-pomxml" class="hash-link" aria-label="Direct link to 6-configure-the-maven-deploy-plugin-in-pomxml" title="Direct link to 6-configure-the-maven-deploy-plugin-in-pomxml"></a></h3>
+<p>Make sure the <code>maven-deploy-plugin</code> is correctly configured in your <code>pom.xml</code>. Here's an example configuration:</p>
+<div class="language-xml codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-xml codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token tag punctuation" style="color:#393A34"><</span><span class="token tag" style="color:#00009f">plug [...]
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="7-increase-verbose-logging">7. Increase Verbose Logging<a href="#7-increase-verbose-logging" class="hash-link" aria-label="Direct link to 7. Increase Verbose Logging" title="Direct link to 7. Increase Verbose Logging"></a></h3>
+<p>Enable verbose logging to get more details about the error. You can do this by adding the <code>-X</code> flag when running the Maven command:</p>
+<div class="language-sh codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-sh codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn clean deploy </span><span class="token parameter variable" style="color:#36acaa">-X</span>< [...]
+<p><strong>[DONE]</strong></p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="8-retry-with-a-clean-local-repository">8. Retry with a Clean Local Repository<a href="#8-retry-with-a-clean-local-repository" class="hash-link" aria-label="Direct link to 8. Retry with a Clean Local Repository" title="Direct link to 8. Retry with a Clean Local Repository"></a></h3>
+<p>Sometimes, a corrupt local repository can cause issues. Try cleaning your local Maven repository and re-running the deployment:</p>
+<div class="language-sh codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-sh codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">mvn clean </span><span class="token function" style="color:#d73a49">install</span><span class=" [...]
+<p><em>[OPEN]</em> - Does not match to an authentication issue.</p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="9-check-for-missing-credentials">9. Check for Missing Credentials<a href="#9-check-for-missing-credentials" class="hash-link" aria-label="Direct link to 9. Check for Missing Credentials" title="Direct link to 9. Check for Missing Credentials"></a></h3>
+<p>Ensure that the credentials for the repository are correctly set up in your <code>settings.xml</code>:</p>
+<div class="language-xml codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-xml codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token tag punctuation" style="color:#393A34"><</span><span class="token tag" style="color:#00009f">serv [...]
+<p><strong>[DONE]</strong></p>
+<h3 class="anchor anchorWithStickyNavbar_LWe7" id="10-review-the-full-stack-trace">10. Review the Full Stack Trace<a href="#10-review-the-full-stack-trace" class="hash-link" aria-label="Direct link to 10. Review the Full Stack Trace" title="Direct link to 10. Review the Full Stack Trace"></a></h3>
+<p>The full stack trace indicates a <code>NullPointerException</code>:</p>
+<div class="language-plaintext codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-plaintext codeBlock_bY9V thin-scrollbar" style="color:#393A34;background-color:#f6f8fa"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">Caused by: java.lang.NullPointerException</span><br></span><span class="token-lin [...]
+<p>This suggests that there might be an issue with how credentials are being handled. Double-check that the credentials are being correctly passed and processed.</p>
+<p><strong>[DONE]</strong></p></div><footer class="row docusaurus-mt-lg blogPostFooterDetailsFull_mRVl"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/asf">ASF</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/release">release</a></li></ul></div></footer> [...]
+ <p> Apache Wayang is an effort undergoing incubation at The Apache Software Foundation (ASF), sponsored by the Apache Incubator. Incubation is required of all newly accepted projects until a further review indicates that the infrastructure, communications, and decision making process have stabilized in a manner consistent with other successful ASF projects. While incubation status is not necessarily a reflection of the completeness or stability of the code, it does indicate that th [...]
+ <p>
+ Copyright © 2024 The Apache Software Foundation, Licensed under the Apache License, Version 2.0. <br>
+ Apache, the names of Apache projects, and the feather logo are either registered trademarks or trademarks of the Apache Software Foundation in the United States and/or other countries.
+ </p>
+ </div></div></div></div></footer></div>
+</body>
+</html>
\ No newline at end of file
diff --git a/blog/wayang-tensorflow/index.html b/blog/wayang-tensorflow/index.html
index ce7a3843..5353ea1e 100644
--- a/blog/wayang-tensorflow/index.html
+++ b/blog/wayang-tensorflow/index.html
@@ -5,11 +5,11 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">Integrating ML platforms in Wayang | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/blog/wayang-tensorflow"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docusau [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.01bb8940.js" defer="defer"></script>
-<script src="/assets/js/main.0cc2e93a.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.8489d39d.js" defer="defer"></script>
+<script src="/assets/js/main.4f5176d9.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
-<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
+<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
Thanks to the extensible nature of Wayang, the only core changes we had to do were introducing the concept of a <code>Model</code> and implement a new driver for the newly added platform.</p>
<h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-1-introducing-a-model">Step 1: Introducing a Model<a href="#step-1-introducing-a-model" class="hash-link" aria-label="Direct link to Step 1: Introducing a Model" title="Direct link to Step 1: Introducing a Model"></a></h2>
<p>With respect to the model, we followed Wayang’s abstraction philosophy: We created a <code>Model</code> interface to be used as input or output by Wayang operators and then extended it for the platform-specific operators. Different model interfaces can be found here:</p>
@@ -41,7 +41,7 @@ Thanks to the extensible nature of Wayang, the only core changes we had to do we
coming soon (including the federated learning aspect as well as an SQL interface and a novel
data debugging functionality). If you want to hear or join the community, consult the link
<a href="https://wayang.apache.org/community/" target="_blank" rel="noopener noreferrer">https://wayang.apache.org/community/</a> , join the mailing lists, contribute with new ideas,
-write documentation, or fix bugs.</p></div><footer class="row docusaurus-mt-lg blogPostFooterDetailsFull_mRVl"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/ml">ML</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/tensorflow">tensorflow</a></li></ul></di [...]
+write documentation, or fix bugs.</p></div><footer class="row docusaurus-mt-lg blogPostFooterDetailsFull_mRVl"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/ml">ML</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/tensorflow">tensorflow</a></li></ul></di [...]
<p> Apache Wayang is an effort undergoing incubation at The Apache Software Foundation (ASF), sponsored by the Apache Incubator. Incubation is required of all newly accepted projects until a further review indicates that the infrastructure, communications, and decision making process have stabilized in a manner consistent with other successful ASF projects. While incubation status is not necessarily a reflection of the completeness or stability of the code, it does indicate that th [...]
<p>
Copyright © 2024 The Apache Software Foundation, Licensed under the Apache License, Version 2.0. <br>
diff --git a/blog/wayang-vs-trino/index.html b/blog/wayang-vs-trino/index.html
index 7953ea12..1a1ef9f4 100644
--- a/blog/wayang-vs-trino/index.html
+++ b/blog/wayang-vs-trino/index.html
@@ -5,11 +5,11 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">Apache Wayang vs. Presto/Trino | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/blog/wayang-vs-trino"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docusaurus_ta [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.01bb8940.js" defer="defer"></script>
-<script src="/assets/js/main.0cc2e93a.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.8489d39d.js" defer="defer"></script>
+<script src="/assets/js/main.4f5176d9.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
-<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
+<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
<h2 class="anchor anchorWithStickyNavbar_LWe7" id="key-distinctions">Key Distinctions<a href="#key-distinctions" class="hash-link" aria-label="Direct link to Key Distinctions" title="Direct link to Key Distinctions"></a></h2>
<p>Trino/Presto is a <strong>query engine</strong> for <strong>distributed SQL query processing</strong>. It is composed of a coordinator and multiple workers. The coordinator consists of a query optimizer and a scheduler, while the workers are responsible for performing the necessary query processing. Data is fetched from external systems via a Connector API, i.e., Trino/Presto supports <a href="https://trino.io/ecosystem/data-source" target="_blank" rel="noopener noreferrer">multiple d [...]
<p>In contrast, Wayang is a <strong>middleware</strong> for <strong>integrating diverse data platforms</strong>, including but not limited to query engines. This means that Wayang leverages the processing capabilities of the underlying data platforms to complete a given job, with no actual query processing taking place within Wayang itself.</p>
@@ -44,7 +44,7 @@ In fact, Trino can be easily plugged to Wayang as a platform and be seamlessly i
</ul>
<h2 class="anchor anchorWithStickyNavbar_LWe7" id="conclusion">Conclusion<a href="#conclusion" class="hash-link" aria-label="Direct link to Conclusion" title="Direct link to Conclusion"></a></h2>
<p>Trino is a distributed SQL query engine which performs all the query processing of an input SQL query in a distributed manner. Wayang, on the other hand, is a data platform integrator which can automatically determine which data platform(s) is best suited for an application.</p>
-<p>Author: <a href="https://github.com/zkaoudi" target="_blank" rel="noopener noreferrer">zkaoudi</a></p></div><footer class="row docusaurus-mt-lg blogPostFooterDetailsFull_mRVl"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/presto">presto</a></li><li class="tag_QGVx"><a class="tag_zVe [...]
+<p>Author: <a href="https://github.com/zkaoudi" target="_blank" rel="noopener noreferrer">zkaoudi</a></p></div><footer class="row docusaurus-mt-lg blogPostFooterDetailsFull_mRVl"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/presto">presto</a></li><li class="tag_QGVx"><a class="tag_zVe [...]
<p> Apache Wayang is an effort undergoing incubation at The Apache Software Foundation (ASF), sponsored by the Apache Incubator. Incubation is required of all newly accepted projects until a further review indicates that the infrastructure, communications, and decision making process have stabilized in a manner consistent with other successful ASF projects. While incubation status is not necessarily a reflection of the completeness or stability of the code, it does indicate that th [...]
<p>
Copyright © 2024 The Apache Software Foundation, Licensed under the Apache License, Version 2.0. <br>
diff --git a/blog/website_update/index.html b/blog/website_update/index.html
index 916ecb62..cd2acb6a 100644
--- a/blog/website_update/index.html
+++ b/blog/website_update/index.html
@@ -5,11 +5,11 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">Website updated | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/blog/website_update"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docusaurus_tag" content="defa [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.01bb8940.js" defer="defer"></script>
-<script src="/assets/js/main.0cc2e93a.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.8489d39d.js" defer="defer"></script>
+<script src="/assets/js/main.4f5176d9.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
-<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
+<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
<p>Author: <a href="https://github.com/2pk03" target="_blank" rel="noopener noreferrer">2pk03</a></p>
<p>We switched to a new CMS. That's all.</p>
<h2 class="anchor anchorWithStickyNavbar_LWe7" id="cheatsheet">Cheatsheet<a href="#cheatsheet" class="hash-link" aria-label="Direct link to Cheatsheet" title="Direct link to Cheatsheet"></a></h2>
@@ -39,7 +39,7 @@
<li>Line one</li>
<li>Line two</li>
<li>Line three</li>
-</ul></div><footer class="row docusaurus-mt-lg blogPostFooterDetailsFull_mRVl"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li></ul></div></footer></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Blog post page navigation"><a class="pagination-nav__link pagination-nav__link--prev" href="/blog/kafka-meets-wayang-1"><div class="pagination-nav__ [...]
+</ul></div><footer class="row docusaurus-mt-lg blogPostFooterDetailsFull_mRVl"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/wayang">wayang</a></li></ul></div></footer></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Blog post page navigation"><a class="pagination-nav__link pagination-nav__link--prev" href="/blog/kafka-meets-wayang-1"><div class="pagination-nav__ [...]
<p> Apache Wayang is an effort undergoing incubation at The Apache Software Foundation (ASF), sponsored by the Apache Incubator. Incubation is required of all newly accepted projects until a further review indicates that the infrastructure, communications, and decision making process have stabilized in a manner consistent with other successful ASF projects. While incubation status is not necessarily a reflection of the completeness or stability of the code, it does indicate that th [...]
<p>
Copyright © 2024 The Apache Software Foundation, Licensed under the Apache License, Version 2.0. <br>
diff --git a/docs/community/committer/index.html b/docs/community/committer/index.html
index a86d620b..47ea05b9 100644
--- a/docs/community/committer/index.html
+++ b/docs/community/committer/index.html
@@ -5,8 +5,8 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">Becoming a committer | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/docs/community/committer"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language" [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.01bb8940.js" defer="defer"></script>
-<script src="/assets/js/main.0cc2e93a.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.8489d39d.js" defer="defer"></script>
+<script src="/assets/js/main.4f5176d9.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
diff --git a/docs/community/contribute/index.html b/docs/community/contribute/index.html
index 66fbdd99..48732c6c 100644
--- a/docs/community/contribute/index.html
+++ b/docs/community/contribute/index.html
@@ -5,8 +5,8 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">How to contribute | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/docs/community/contribute"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language" c [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.01bb8940.js" defer="defer"></script>
-<script src="/assets/js/main.0cc2e93a.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.8489d39d.js" defer="defer"></script>
+<script src="/assets/js/main.4f5176d9.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
diff --git a/docs/community/mailinglist/index.html b/docs/community/mailinglist/index.html
index 2d5a5f29..57585664 100644
--- a/docs/community/mailinglist/index.html
+++ b/docs/community/mailinglist/index.html
@@ -5,8 +5,8 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">Mailinglists | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/docs/community/mailinglist"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language" conte [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.01bb8940.js" defer="defer"></script>
-<script src="/assets/js/main.0cc2e93a.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.8489d39d.js" defer="defer"></script>
+<script src="/assets/js/main.4f5176d9.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
diff --git a/docs/community/repositories/index.html b/docs/community/repositories/index.html
index 06ea3b26..62f804ba 100644
--- a/docs/community/repositories/index.html
+++ b/docs/community/repositories/index.html
@@ -5,8 +5,8 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">Repositories | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/docs/community/repositories"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language" cont [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.01bb8940.js" defer="defer"></script>
-<script src="/assets/js/main.0cc2e93a.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.8489d39d.js" defer="defer"></script>
+<script src="/assets/js/main.4f5176d9.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
diff --git a/docs/community/team/index.html b/docs/community/team/index.html
index 3f28b88c..f915172f 100644
--- a/docs/community/team/index.html
+++ b/docs/community/team/index.html
@@ -5,8 +5,8 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">Team | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/docs/community/team"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language" content="en"><meta d [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.01bb8940.js" defer="defer"></script>
-<script src="/assets/js/main.0cc2e93a.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.8489d39d.js" defer="defer"></script>
+<script src="/assets/js/main.4f5176d9.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
diff --git a/docs/guide/adding-operators/index.html b/docs/guide/adding-operators/index.html
index 62ba293b..26d91f0f 100644
--- a/docs/guide/adding-operators/index.html
+++ b/docs/guide/adding-operators/index.html
@@ -5,8 +5,8 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">Adding an operator in Wayang | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/docs/guide/adding-operators"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearc [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.01bb8940.js" defer="defer"></script>
-<script src="/assets/js/main.0cc2e93a.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.8489d39d.js" defer="defer"></script>
+<script src="/assets/js/main.4f5176d9.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
diff --git a/docs/guide/api-documentation/index.html b/docs/guide/api-documentation/index.html
index b68b3188..9d44452d 100644
--- a/docs/guide/api-documentation/index.html
+++ b/docs/guide/api-documentation/index.html
@@ -5,8 +5,8 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">API documentation | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/docs/guide/api-documentation"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.01bb8940.js" defer="defer"></script>
-<script src="/assets/js/main.0cc2e93a.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.8489d39d.js" defer="defer"></script>
+<script src="/assets/js/main.4f5176d9.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
diff --git a/docs/guide/developing-in-wayang/index.html b/docs/guide/developing-in-wayang/index.html
index 7b6752a4..3548e36f 100644
--- a/docs/guide/developing-in-wayang/index.html
+++ b/docs/guide/developing-in-wayang/index.html
@@ -5,8 +5,8 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">Developing in Wayang | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/docs/guide/developing-in-wayang"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:la [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.01bb8940.js" defer="defer"></script>
-<script src="/assets/js/main.0cc2e93a.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.8489d39d.js" defer="defer"></script>
+<script src="/assets/js/main.4f5176d9.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
diff --git a/docs/guide/examples/index.html b/docs/guide/examples/index.html
index e181fdd7..976acad0 100644
--- a/docs/guide/examples/index.html
+++ b/docs/guide/examples/index.html
@@ -5,8 +5,8 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">Usage Examples | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/docs/guide/examples"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language" content="e [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.01bb8940.js" defer="defer"></script>
-<script src="/assets/js/main.0cc2e93a.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.8489d39d.js" defer="defer"></script>
+<script src="/assets/js/main.4f5176d9.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
diff --git a/docs/guide/getting-started/index.html b/docs/guide/getting-started/index.html
index 4c37fe44..dedae80b 100644
--- a/docs/guide/getting-started/index.html
+++ b/docs/guide/getting-started/index.html
@@ -5,8 +5,8 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">Getting started | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/docs/guide/getting-started"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language" co [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.01bb8940.js" defer="defer"></script>
-<script src="/assets/js/main.0cc2e93a.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.8489d39d.js" defer="defer"></script>
+<script src="/assets/js/main.4f5176d9.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
diff --git a/docs/guide/installation/index.html b/docs/guide/installation/index.html
index bda656c0..8982a540 100644
--- a/docs/guide/installation/index.html
+++ b/docs/guide/installation/index.html
@@ -5,8 +5,8 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">How to build Wayang | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/docs/guide/installation"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language" c [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.01bb8940.js" defer="defer"></script>
-<script src="/assets/js/main.0cc2e93a.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.8489d39d.js" defer="defer"></script>
+<script src="/assets/js/main.4f5176d9.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
diff --git a/docs/guide/ml4all/index.html b/docs/guide/ml4all/index.html
index c7458356..77220eec 100644
--- a/docs/guide/ml4all/index.html
+++ b/docs/guide/ml4all/index.html
@@ -5,8 +5,8 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">Scalable Machine Learning | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/docs/guide/ml4all"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language" c [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.01bb8940.js" defer="defer"></script>
-<script src="/assets/js/main.0cc2e93a.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.8489d39d.js" defer="defer"></script>
+<script src="/assets/js/main.4f5176d9.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
diff --git a/docs/guide/optimizer/index.html b/docs/guide/optimizer/index.html
index 9b2ff787..aa391ec0 100644
--- a/docs/guide/optimizer/index.html
+++ b/docs/guide/optimizer/index.html
@@ -5,8 +5,8 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">Data processing optimizer | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/docs/guide/optimizer"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.01bb8940.js" defer="defer"></script>
-<script src="/assets/js/main.0cc2e93a.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.8489d39d.js" defer="defer"></script>
+<script src="/assets/js/main.4f5176d9.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
diff --git a/docs/guide/wayang-docker/index.html b/docs/guide/wayang-docker/index.html
index 08672299..6be4ef66 100644
--- a/docs/guide/wayang-docker/index.html
+++ b/docs/guide/wayang-docker/index.html
@@ -5,8 +5,8 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">Using Wayang's docker image | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/docs/guide/wayang-docker"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsear [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.01bb8940.js" defer="defer"></script>
-<script src="/assets/js/main.0cc2e93a.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.8489d39d.js" defer="defer"></script>
+<script src="/assets/js/main.4f5176d9.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
diff --git a/docs/introduction/about/index.html b/docs/introduction/about/index.html
index 18320a50..6be4c8c1 100644
--- a/docs/introduction/about/index.html
+++ b/docs/introduction/about/index.html
@@ -5,8 +5,8 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">What is Wayang? | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/docs/introduction/about"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language" conte [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.01bb8940.js" defer="defer"></script>
-<script src="/assets/js/main.0cc2e93a.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.8489d39d.js" defer="defer"></script>
+<script src="/assets/js/main.4f5176d9.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
diff --git a/docs/introduction/benchmark/index.html b/docs/introduction/benchmark/index.html
index 83128f33..e70a7d58 100644
--- a/docs/introduction/benchmark/index.html
+++ b/docs/introduction/benchmark/index.html
@@ -5,8 +5,8 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">Benchmarking Wayang | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/docs/introduction/benchmark"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:languag [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.01bb8940.js" defer="defer"></script>
-<script src="/assets/js/main.0cc2e93a.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.8489d39d.js" defer="defer"></script>
+<script src="/assets/js/main.4f5176d9.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
diff --git a/docs/introduction/features/index.html b/docs/introduction/features/index.html
index 1a242a54..aba78571 100644
--- a/docs/introduction/features/index.html
+++ b/docs/introduction/features/index.html
@@ -5,8 +5,8 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">Features | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/docs/introduction/features"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language" content=" [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.01bb8940.js" defer="defer"></script>
-<script src="/assets/js/main.0cc2e93a.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.8489d39d.js" defer="defer"></script>
+<script src="/assets/js/main.4f5176d9.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
diff --git a/docs/start/download/index.html b/docs/start/download/index.html
index 88d9274e..37e82126 100644
--- a/docs/start/download/index.html
+++ b/docs/start/download/index.html
@@ -5,8 +5,8 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">Download Apache Wayang | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/docs/start/download"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language" co [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.01bb8940.js" defer="defer"></script>
-<script src="/assets/js/main.0cc2e93a.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.8489d39d.js" defer="defer"></script>
+<script src="/assets/js/main.4f5176d9.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
diff --git a/index.html b/index.html
index 1938ec19..39c358a8 100644
--- a/index.html
+++ b/index.html
@@ -5,8 +5,8 @@
<meta name="generator" content="Docusaurus v3.1.1">
<title data-rh="true">Apache Wayang (incubating) | Apache Wayang (incubating)</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://wayang.apache.org/"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docusaurus_tag" content="default"><me [...]
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Wayang (incubating) Atom Feed"><link rel="stylesheet" href="/assets/css/styles.ecf70413.css">
-<script src="/assets/js/runtime~main.01bb8940.js" defer="defer"></script>
-<script src="/assets/js/main.0cc2e93a.js" defer="defer"></script>
+<script src="/assets/js/runtime~main.8489d39d.js" defer="defer"></script>
+<script src="/assets/js/main.4f5176d9.js" defer="defer"></script>
</head>
<body class="navigation-with-keyboard">
<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return localStorage.getItem("theme")}catch(t){}}();t(null!==e?e:"light")}(),function(){try{const a=new URLSearchParams(window.location.search).entries();for(var[t,e]of a)if(t.startsWith("docusaurus-data-")){var n=t.replace("docusaurus-data-","data-");document.documentElement.se [...]
diff --git a/lunr-index-1726903642488.json b/lunr-index-1726903642488.json
deleted file mode 100644
index 57ba5f7d..00000000
--- a/lunr-index-1726903642488.json
+++ /dev/null
@@ -1 +0,0 @@
-{"version":"2.3.9","fields":["title","content","keywords"],"fieldVectors":[["title/0",[0,140.361,1,250.285,2,435.577,3,30.862,4,176.87,5,274.264,6,236.613]],["content/0",[]],["keywords/0",[]],["title/1",[7,1237.134]],["content/1",[0,2.212,1,3.133,3,0.577,5,6.798,8,7.577,9,5.296,10,5.955,11,5.076,12,6.559,13,4.581,14,10.635,15,3.067,16,5.685,17,6.719,18,7.577,19,6.719,20,7.069,21,6.719,22,6.719,23,5.076,24,3.548,25,6.756,26,8.083,27,5.076,28,4.776,29,3.704,30,2.669,31,1.79,32,3.673,33,2.4 [...]
\ No newline at end of file
diff --git a/lunr-index-1727001579936.json b/lunr-index-1727001579936.json
new file mode 100644
index 00000000..dcd98108
--- /dev/null
+++ b/lunr-index-1727001579936.json
@@ -0,0 +1 @@
+{"version":"2.3.9","fields":["title","content","keywords"],"fieldVectors":[["title/0",[0,118.833,1,324.362,2,548.246,3,51.103,4,353.336,5,273.302]],["content/0",[]],["keywords/0",[]],["title/1",[6,1142.994]],["content/1",[0,2.475,1,3.464,3,0.808,4,7.358,7,7.639,8,5.588,9,6.373,10,5.467,11,7.091,12,5.193,13,11.242,14,3.292,15,6.162,16,7.161,17,8.097,18,7.161,19,7.611,20,7.161,21,7.161,22,5.158,23,3.773,24,7.257,25,7.913,26,5.158,27,5.158,28,4.136,29,2.733,30,2.005,31,3.773,32,2.733,33,2.4 [...]
\ No newline at end of file
diff --git a/lunr-index.json b/lunr-index.json
index 57ba5f7d..dcd98108 100644
--- a/lunr-index.json
+++ b/lunr-index.json
@@ -1 +1 @@
-{"version":"2.3.9","fields":["title","content","keywords"],"fieldVectors":[["title/0",[0,140.361,1,250.285,2,435.577,3,30.862,4,176.87,5,274.264,6,236.613]],["content/0",[]],["keywords/0",[]],["title/1",[7,1237.134]],["content/1",[0,2.212,1,3.133,3,0.577,5,6.798,8,7.577,9,5.296,10,5.955,11,5.076,12,6.559,13,4.581,14,10.635,15,3.067,16,5.685,17,6.719,18,7.577,19,6.719,20,7.069,21,6.719,22,6.719,23,5.076,24,3.548,25,6.756,26,8.083,27,5.076,28,4.776,29,3.704,30,2.669,31,1.79,32,3.673,33,2.4 [...]
\ No newline at end of file
+{"version":"2.3.9","fields":["title","content","keywords"],"fieldVectors":[["title/0",[0,118.833,1,324.362,2,548.246,3,51.103,4,353.336,5,273.302]],["content/0",[]],["keywords/0",[]],["title/1",[6,1142.994]],["content/1",[0,2.475,1,3.464,3,0.808,4,7.358,7,7.639,8,5.588,9,6.373,10,5.467,11,7.091,12,5.193,13,11.242,14,3.292,15,6.162,16,7.161,17,8.097,18,7.161,19,7.611,20,7.161,21,7.161,22,5.158,23,3.773,24,7.257,25,7.913,26,5.158,27,5.158,28,4.136,29,2.733,30,2.005,31,3.773,32,2.733,33,2.4 [...]
\ No newline at end of file
diff --git a/search-doc-1726903642488.json b/search-doc-1726903642488.json
deleted file mode 100644
index 30d37053..00000000
--- a/search-doc-1726903642488.json
+++ /dev/null
@@ -1 +0,0 @@
-{"searchDocs":[{"title":"Apache Kafka meets Apache Wayang : Part 1","type":0,"sectionRef":"#","url":"/blog/kafka-meets-wayang-1","content":"","keywords":"","version":null},{"title":"Intro","type":1,"pageTitle":"Apache Kafka meets Apache Wayang : Part 1","url":"/blog/kafka-meets-wayang-1#intro","content":" This article is the first of a four part series about federated data analysis using Apache Wayang. The first article starts with an introduction of a typical data colaboration scenario [...]
\ No newline at end of file
diff --git a/search-doc-1727001579936.json b/search-doc-1727001579936.json
new file mode 100644
index 00000000..f77ff216
--- /dev/null
+++ b/search-doc-1727001579936.json
@@ -0,0 +1 @@
+{"searchDocs":[{"title":"Apache Kafka meets Wayang - Part 1","type":0,"sectionRef":"#","url":"/blog/kafka-meets-wayang-1","content":"","keywords":"","version":null},{"title":"Intro","type":1,"pageTitle":"Apache Kafka meets Wayang - Part 1","url":"/blog/kafka-meets-wayang-1#intro","content":" This article is the first of a four part series about federated data analysis using Apache Wayang. The first article starts with an introduction of a typical data colaboration scenario which will em [...]
\ No newline at end of file
diff --git a/search-doc.json b/search-doc.json
index 30d37053..f77ff216 100644
--- a/search-doc.json
+++ b/search-doc.json
@@ -1 +1 @@
-{"searchDocs":[{"title":"Apache Kafka meets Apache Wayang : Part 1","type":0,"sectionRef":"#","url":"/blog/kafka-meets-wayang-1","content":"","keywords":"","version":null},{"title":"Intro","type":1,"pageTitle":"Apache Kafka meets Apache Wayang : Part 1","url":"/blog/kafka-meets-wayang-1#intro","content":" This article is the first of a four part series about federated data analysis using Apache Wayang. The first article starts with an introduction of a typical data colaboration scenario [...]
\ No newline at end of file
+{"searchDocs":[{"title":"Apache Kafka meets Wayang - Part 1","type":0,"sectionRef":"#","url":"/blog/kafka-meets-wayang-1","content":"","keywords":"","version":null},{"title":"Intro","type":1,"pageTitle":"Apache Kafka meets Wayang - Part 1","url":"/blog/kafka-meets-wayang-1#intro","content":" This article is the first of a four part series about federated data analysis using Apache Wayang. The first article starts with an introduction of a typical data colaboration scenario which will em [...]
\ No newline at end of file
diff --git a/sitemap.xml b/sitemap.xml
index c9761e1a..9ed503f9 100644
--- a/sitemap.xml
+++ b/sitemap.xml
@@ -1 +1 @@
-<?xml version="1.0" encoding="UTF-8"?><urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:news="http://www.google.com/schemas/sitemap-news/0.9" xmlns:xhtml="http://www.w3.org/1999/xhtml" xmlns:image="http://www.google.com/schemas/sitemap-image/1.1" xmlns:video="http://www.google.com/schemas/sitemap-video/1.1"><url><loc>https://wayang.apache.org/blog</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://wayang.apache.org/blog/archive</loc><chang [...]
\ No newline at end of file
+<?xml version="1.0" encoding="UTF-8"?><urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:news="http://www.google.com/schemas/sitemap-news/0.9" xmlns:xhtml="http://www.w3.org/1999/xhtml" xmlns:image="http://www.google.com/schemas/sitemap-image/1.1" xmlns:video="http://www.google.com/schemas/sitemap-video/1.1"><url><loc>https://wayang.apache.org/blog</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://wayang.apache.org/blog/archive</loc><chang [...]
\ No newline at end of file