You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@wayang.apache.org by be...@apache.org on 2021/05/04 15:00:57 UTC
[incubator-wayang] branch debugger updated (68d4dc0 -> dea33ea)
This is an automated email from the ASF dual-hosted git repository.
bertty pushed a change to branch debugger
in repository https://gitbox.apache.org/repos/asf/incubator-wayang.git.
from 68d4dc0 Merge branch 'main' into debugger
new 1a88f89 [WAYANG-28] Add javadoc to identifier package
new 6fddce1 [WAYANG-28] Add javadoc to iterator package
new c5005d1 [WAYANG-28] Add javadoc to tuple package
new 3a56f9b [WAYANG-28] rename wayang-hackit-shipper-rabbitmq to wayang-hackit-shipper-kafka
new dea33ea [WAYANG-28] creation module wayang-hackit-sidecar
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
wayang-plugins/wayang-hackit/pom.xml | 1 +
.../plugin/hackit/core/identifiers/HackitID.java | 3 +
.../hackit/core/identifiers/HackitIDGenerator.java | 43 +++++++-
.../generator/DistributeSequencial.java | 13 +++
.../{TwitterSnowflake.java => SnowflakeID.java} | 39 +++++--
.../hackit/core/iterator/ElementPlusIterator.java | 25 +++++
.../hackit/core/iterator/FunctionIterator.java | 23 ++++
.../hackit/core/iterator/HackitIterator.java | 13 ++-
.../hackit/core/iterator/OneElementIterator.java | 41 ++++++++
.../plugin/hackit/core/tuple/HackitTuple.java | 67 ++++++++++++
.../plugin/hackit/core/tuple/header/Header.java | 116 +++++++++++++++++++--
.../hackit/core/tuple/header/HeaderBuilder.java | 13 +++
.../hackit/core/tuple/header/HeaderLong.java | 32 +++++-
.../wayang-hackit/wayang-hackit-shipper/pom.xml | 2 +-
.../pom.xml | 2 +-
.../pom.xml | 16 ++-
.../wayang/hackit/sidecar/HackitSidecar.java | 23 ++++
.../wayang/hackit/sidecar/HackitSidecarTest.java | 31 ++++++
18 files changed, 468 insertions(+), 35 deletions(-)
rename wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/identifiers/generator/{TwitterSnowflake.java => SnowflakeID.java} (75%)
rename wayang-plugins/wayang-hackit/wayang-hackit-shipper/{wayang-hackit-shipper-rabbitmq => wayang-hackit-shipper-kafka}/pom.xml (96%)
copy wayang-plugins/wayang-hackit/{wayang-hackit-core => wayang-hackit-sidecar}/pom.xml (79%)
create mode 100644 wayang-plugins/wayang-hackit/wayang-hackit-sidecar/src/main/java/org/apache/wayang/hackit/sidecar/HackitSidecar.java
create mode 100644 wayang-plugins/wayang-hackit/wayang-hackit-sidecar/src/test/java/org/apache/wayang/hackit/sidecar/HackitSidecarTest.java
[incubator-wayang] 05/05: [WAYANG-28] creation module
wayang-hackit-sidecar
Posted by be...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bertty pushed a commit to branch debugger
in repository https://gitbox.apache.org/repos/asf/incubator-wayang.git
commit dea33ea7b387411f2920950167c103f22c532351
Author: Bertty Contreras-Rojas <be...@scalytics.io>
AuthorDate: Tue May 4 11:00:34 2021 -0400
[WAYANG-28] creation module wayang-hackit-sidecar
---
wayang-plugins/wayang-hackit/pom.xml | 1 +
.../{ => wayang-hackit-sidecar}/pom.xml | 19 +++----------
.../wayang/hackit/sidecar/HackitSidecar.java | 23 ++++++++++++++++
.../wayang/hackit/sidecar/HackitSidecarTest.java | 31 ++++++++++++++++++++++
4 files changed, 59 insertions(+), 15 deletions(-)
diff --git a/wayang-plugins/wayang-hackit/pom.xml b/wayang-plugins/wayang-hackit/pom.xml
index d86fb5d..b3e914f 100644
--- a/wayang-plugins/wayang-hackit/pom.xml
+++ b/wayang-plugins/wayang-hackit/pom.xml
@@ -39,6 +39,7 @@
<modules>
<module>wayang-hackit-core</module>
<module>wayang-hackit-shipper</module>
+ <module>wayang-hackit-sidecar</module>
</modules>
<properties>
diff --git a/wayang-plugins/wayang-hackit/pom.xml b/wayang-plugins/wayang-hackit/wayang-hackit-sidecar/pom.xml
similarity index 74%
copy from wayang-plugins/wayang-hackit/pom.xml
copy to wayang-plugins/wayang-hackit/wayang-hackit-sidecar/pom.xml
index d86fb5d..64ba661 100644
--- a/wayang-plugins/wayang-hackit/pom.xml
+++ b/wayang-plugins/wayang-hackit/wayang-hackit-sidecar/pom.xml
@@ -21,28 +21,17 @@
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
- <artifactId>wayang-plugins</artifactId>
+ <artifactId>wayang</artifactId>
<groupId>org.apache.wayang</groupId>
<version>0.6.0-SNAPSHOT</version>
</parent>
-
<modelVersion>4.0.0</modelVersion>
- <artifactId>wayang-hackit</artifactId>
- <packaging>pom</packaging>
-
- <name>Wayang Hackit</name>
-
- <description>
- This Wayang module contains the implementation of the paper https://wayang.apache.org/assets/pdf/paper/socc19.pdf
- </description>
- <modules>
- <module>wayang-hackit-core</module>
- <module>wayang-hackit-shipper</module>
- </modules>
+ <artifactId>wayang-hackit-sidecar</artifactId>
<properties>
-
+ <maven.compiler.source>11</maven.compiler.source>
+ <maven.compiler.target>11</maven.compiler.target>
</properties>
</project>
diff --git a/wayang-plugins/wayang-hackit/wayang-hackit-sidecar/src/main/java/org/apache/wayang/hackit/sidecar/HackitSidecar.java b/wayang-plugins/wayang-hackit/wayang-hackit-sidecar/src/main/java/org/apache/wayang/hackit/sidecar/HackitSidecar.java
new file mode 100644
index 0000000..7200fa4
--- /dev/null
+++ b/wayang-plugins/wayang-hackit/wayang-hackit-sidecar/src/main/java/org/apache/wayang/hackit/sidecar/HackitSidecar.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.wayang.hackit.sidecar;
+
+public class HackitSidecar {
+}
diff --git a/wayang-plugins/wayang-hackit/wayang-hackit-sidecar/src/test/java/org/apache/wayang/hackit/sidecar/HackitSidecarTest.java b/wayang-plugins/wayang-hackit/wayang-hackit-sidecar/src/test/java/org/apache/wayang/hackit/sidecar/HackitSidecarTest.java
new file mode 100644
index 0000000..b2c53b8
--- /dev/null
+++ b/wayang-plugins/wayang-hackit/wayang-hackit-sidecar/src/test/java/org/apache/wayang/hackit/sidecar/HackitSidecarTest.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.wayang.hackit.sidecar;
+
+import org.junit.jupiter.api.BeforeEach;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+class HackitSidecarTest {
+
+ @BeforeEach
+ void setUp() {
+ }
+}
[incubator-wayang] 03/05: [WAYANG-28] Add javadoc to tuple package
Posted by be...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bertty pushed a commit to branch debugger
in repository https://gitbox.apache.org/repos/asf/incubator-wayang.git
commit c5005d1ab5171b692ef33af9cba8859aaa36e372
Author: Bertty Contreras-Rojas <be...@scalytics.io>
AuthorDate: Fri Apr 30 00:24:47 2021 -0400
[WAYANG-28] Add javadoc to tuple package
---
.../plugin/hackit/core/tuple/HackitTuple.java | 67 ++++++++++++
.../plugin/hackit/core/tuple/header/Header.java | 116 +++++++++++++++++++--
.../hackit/core/tuple/header/HeaderBuilder.java | 13 +++
.../hackit/core/tuple/header/HeaderLong.java | 32 +++++-
4 files changed, 216 insertions(+), 12 deletions(-)
diff --git a/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/tuple/HackitTuple.java b/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/tuple/HackitTuple.java
index d11ee20..8c2cc51 100644
--- a/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/tuple/HackitTuple.java
+++ b/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/tuple/HackitTuple.java
@@ -25,47 +25,114 @@ import org.apache.wayang.plugin.hackit.core.tuple.header.HeaderBuilder;
import java.io.Serializable;
import java.util.Iterator;
+/**
+ * HackitTuple is the base of Hackit because is the basic structure where the needed elements are added
+ * to enable the execution of the logic in the internal pipeline of hackit
+ *
+ * HackitTuple implements {@link Serializable} because the HackitTuple and the <code>T</code> will sent
+ * out or part of the suffle process
+ *
+ * HackitTuple implements {@link ActionGroup} because it could have any {@link org.apache.wayang.plugin.hackit.core.action.Action}
+ * to perform, and is easy to validate at runtime what are the operation that need to be performed
+ *
+ * @param <K> type of the key that will use as indentifier on the HackitTuple
+ * @param <T> type of the element that it will be wrapper inside of the HackitTuple
+ */
public class HackitTuple<K, T> implements Serializable, ActionGroup {
+
+ /**
+ * BUILDER is the {@link HeaderBuilder} that produce the identifier on the process
+ * of construction the new {@link Header}
+ */
private static HeaderBuilder BUILDER;
+
+ /**
+ * header is an {@link Header}, this help to save relevant meta data of the tuple
+ */
private Header<K> header;
+
+ /**
+ * value is the element that is wrapped
+ */
private T value;
+ /**
+ * this static create the {@link HeaderBuilder} that will be use during the process
+ * of geneating the {@link Header}
+ */
static {
+ //TODO: the generation of the HeaderBuilder need to be done by configuration and using maybe a Dependency Inyection
BUILDER = new HeaderBuilder();
}
+ /**
+ * Default Construct, internally it generate the tuple
+ *
+ * @param value that it will wrapper by the {@link HackitTuple}
+ */
public HackitTuple(T value){
this.header = BUILDER.generateHeader();
this.value = value;
}
+ /**
+ * Construct where the header could be provided an not use the default {@link HeaderBuilder}
+ *
+ * @param header {@link Header} that will be save the relevant metadata
+ * @param value that it will wrapper by the {@link HackitTuple}
+ */
public HackitTuple(Header<K> header, T value){
this.header = header;
this.value = value;
}
+ /**
+ * obtain the header of the tuple
+ *
+ * @return {@link Header} that contains the relevant metadata
+ */
public Header<K> getHeader(){
return this.header;
}
+ /**
+ * obtains the identifier allocated inside of the {@link Header}
+ *
+ * @return identifier of the tuple
+ */
public K getKey(){
return this.header.getId();
}
+ /**
+ * obtain the wrapped element inside of the {@link HackitTuple}
+ *
+ * @return original value that was wrapped
+ */
public T getValue(){
return this.value;
}
+ /**
+ * add an {@link HackitTag} to the {@link Header}
+ *
+ * @param tag {@link HackitTag} that it will need at some point on the process
+ */
public void addTag(HackitTag tag){
this.header.addTag(tag);
}
+ /**
+ * get a {@link Iterator} of the currents {@link HackitTag} that are inside of the {@link Header}
+ * @return {@link Iterator} of tags
+ */
public Iterator<HackitTag> getTags(){
return this.header.iterate();
}
@Override
public String toString() {
+ //TODO: change to String.format
return "HackItTuple{" +
"header=" + header +
", value=" + value +
diff --git a/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/tuple/header/Header.java b/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/tuple/header/Header.java
index 49b15a8..2d3577b 100644
--- a/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/tuple/header/Header.java
+++ b/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/tuple/header/Header.java
@@ -26,78 +26,178 @@ import java.util.Collections;
import java.util.Iterator;
import java.util.List;
+/**
+ * Header is the container of the metadata asociated to one {@link org.apache.wayang.plugin.hackit.core.tuple.HackitTuple}
+ *
+ * @param <K> type of the identifier of the {@link org.apache.wayang.plugin.hackit.core.tuple.HackitTuple}
+ */
public abstract class Header<K> implements Serializable, ActionGroup {
+
+ /**
+ * id is identifier of the header and also the tuple
+ */
private K id;
+
+ /**
+ * child indicate the number of the child that the current header is from the
+ * original {@link org.apache.wayang.plugin.hackit.core.tuple.HackitTuple} and they share the
+ * same identifier <code>id</code>
+ */
protected int child = 0;
+
+ /**
+ * tags added to the header, this describe some action that need to be apply to the
+ * {@link org.apache.wayang.plugin.hackit.core.tuple.HackitTuple}
+ */
private List<HackitTag> tags;
- private boolean has_Callback_tag = false;
+
+ /**
+ * during the process of adding news {@link HackitTag} could add a new {@link org.apache.wayang.plugin.hackit.core.action.Action} at the header, and this
+ * change the status of <code>has_callback_tag</code>
+ */
+ private boolean has_callback_tag = false;
+
+ /**
+ * during the process of adding news {@link HackitTag} could add a new {@link org.apache.wayang.plugin.hackit.core.action.Action} at the header, and this
+ * change the status of <code>has_skip_tag</code>
+ */
private boolean has_skip_tag = false;
+
+ /**
+ * during the process of adding news {@link HackitTag} could add a new {@link org.apache.wayang.plugin.hackit.core.action.Action} at the header, and this
+ * change the status of <code>has_sendout_tag</code>
+ */
private boolean has_sendout_tag = false;
+
+ /**
+ * during the process of adding news {@link HackitTag} could add a new {@link org.apache.wayang.plugin.hackit.core.action.Action} at the header, and this
+ * change the status of <code>has_haltjob_tag</code>
+ */
private boolean has_haltjob_tag = false;
+ /**
+ * Default Construct, this will call {@link #generateID()} and produce the new identifier
+ */
public Header() {
this.id = generateID();
}
+ /**
+ * Construct with the identifier as parameter
+ *
+ * @param id is the identifier of the Header
+ */
public Header(K id) {
this.id = id;
}
+ /**
+ * Construct with the identifier and child identifier as parameter
+ *
+ * @param id is the identifier of the Header
+ * @param child is the child identifier assigned
+ */
public Header(K id, int child){
this(id);
this.child = child;
}
-
+ /**
+ * retrieve the identifier of the Header
+ *
+ * @return current identifier of type <code>K</code>
+ */
public K getId(){
return this.id;
}
+ /**
+ * Add a {@link HackitTag} that could provide a new {@link org.apache.wayang.plugin.hackit.core.action.Action} to be
+ * perfomed by Hackit to the {@link org.apache.wayang.plugin.hackit.core.tuple.HackitTuple}, also update all the
+ * possible action calling the method {@link #updateActionVector(HackitTag)}
+ *
+ * @param tag {@link HackitTag}
+ */
public void addTag(HackitTag tag){
+ //TODO: could be better to use an Set because it just saving uniques elements
if(this.tags == null){
this.tags = new ArrayList<>();
}
this.tags.add(tag);
+ //update all the possible actions on the {@link ActionGroup}
+ //TODO: just execute this action when the element is inserted the first time
updateActionVector(tag);
}
+ /**
+ * remove all the tags from the header, and set all the possible options as false
+ */
public void clearTags(){
this.tags.clear();
- this.has_Callback_tag = false;
+ this.has_callback_tag = false;
this.has_haltjob_tag = false;
this.has_sendout_tag = false;
this.has_skip_tag = false;
}
+ /**
+ * iterate provide an {@link Iterator} that contains all the {@link HackitTag} that were
+ * add on the {@link #tags}
+ *
+ * If the {@link #tags} is null or empty it will return an {@link Collections#emptyIterator()}
+ *
+ * @return {@link Iterator} with the current {@link HackitTag}'s
+ */
public Iterator<HackitTag> iterate(){
+ //TODO: maybe is need to add the option of empty
if(this.tags == null){
return Collections.emptyIterator();
}
return this.tags.iterator();
}
+ /**
+ * Generate a new header that it related on some way with the father header,
+ * depending on the logic of the extender it will be way of generate the child
+ *
+ *
+ * @return new {@link Header} that correspond to the child
+ */
public abstract Header<K> createChild();
+ /**
+ * Generate a new identifier of type <code>K</code> that it will use inside of {@link #Header()}
+ *
+ * @return new identifier of type <code>K</code>
+ */
protected abstract K generateID();
@Override
public String toString() {
+ //TODO: maybe is better to change to String.format
return "HackItTupleHeader{" +
"id=" + id +
", child=" + child +
'}';
}
+ /**
+ * Update the possible {@link org.apache.wayang.plugin.hackit.core.action.Action} that could be perfomed
+ * over the {@link org.apache.wayang.plugin.hackit.core.tuple.HackitTuple}, this is depending of the new
+ * {@link HackitTag}
+ *
+ * @param tag {@link HackitTag} that could have new {@link org.apache.wayang.plugin.hackit.core.action.Action}'s
+ */
private void updateActionVector(HackitTag tag){
- this.has_Callback_tag = (tag.hasCallback())? true: this.has_Callback_tag;
- this.has_haltjob_tag = (tag.isHaltJob())? true: this.has_haltjob_tag;
- this.has_sendout_tag = (tag.isSendOut())? true: this.has_sendout_tag;
- this.has_skip_tag = (tag.isSkip())? true: this.has_skip_tag;
+ this.has_callback_tag = tag.hasCallback() || this.has_callback_tag;
+ this.has_haltjob_tag = tag.isHaltJob() || this.has_haltjob_tag;
+ this.has_sendout_tag = tag.isSendOut() || this.has_sendout_tag;
+ this.has_skip_tag = tag.isSkip() || this.has_skip_tag;
}
@Override
public boolean hasCallback() {
- return this.has_Callback_tag;
+ return this.has_callback_tag;
}
@Override
diff --git a/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/tuple/header/HeaderBuilder.java b/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/tuple/header/HeaderBuilder.java
index 412633f..98ae453 100644
--- a/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/tuple/header/HeaderBuilder.java
+++ b/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/tuple/header/HeaderBuilder.java
@@ -17,14 +17,27 @@
*/
package org.apache.wayang.plugin.hackit.core.tuple.header;
+/**
+ * HeaderBuilder it the generator of {@link Header} to one kind of {@link org.apache.wayang.plugin.hackit.core.tuple.HackitTuple}
+ */
public class HeaderBuilder {
+ /**
+ * Default Construct
+ */
public HeaderBuilder(){
//TODO: take from the configuration
}
+ /**
+ * generate a new Header depending on the configuration provided
+ *
+ * @param <T> is the type that will be provided
+ * @return {@link Header} is new instance of the header requested
+ */
public <T> Header<T> generateHeader(){
+ //TODO: take and works from the configuration provided either on a file or by parameters at runtime
return (Header<T>) new HeaderLong();
}
diff --git a/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/tuple/header/HeaderLong.java b/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/tuple/header/HeaderLong.java
index 9634c4e..2bb6983 100644
--- a/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/tuple/header/HeaderLong.java
+++ b/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/tuple/header/HeaderLong.java
@@ -17,21 +17,45 @@
*/
package org.apache.wayang.plugin.hackit.core.tuple.header;
+/**
+ * HeaderLong extend {@link Header} just a simple implementation of correlative number
+ */
public class HeaderLong extends Header<Long> {
+
+ /**
+ * base is the current correlative that is used in this generator
+ */
static long base;
+ /*
+ TODO: maybe it need to be remove or change to be load form configurations
+ */
static{
base = 0;//(new Random()).nextLong();
}
- public HeaderLong(Long id){
- super(id);
- }
-
+ /**
+ * Default Construct
+ */
public HeaderLong() {
super();
}
+ /**
+ * Construct where is possible to define the identifier
+ *
+ * @param id is the identifier that it will be used by the {@link Header}
+ */
+ public HeaderLong(Long id){
+ super(id);
+ }
+
+ /**
+ * Construct where is possible to define the identifier and child number
+ *
+ * @param id is the identifier that it will be used by the {@link Header}
+ * @param child is the identifier as child of the original element giving by <code>id</code>
+ */
public HeaderLong(Long id, int child) {
super(id, child);
}
[incubator-wayang] 01/05: [WAYANG-28] Add javadoc to identifier
package
Posted by be...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bertty pushed a commit to branch debugger
in repository https://gitbox.apache.org/repos/asf/incubator-wayang.git
commit 1a88f89324b1581f9475a57598d26c6d667405d9
Author: Bertty Contreras-Rojas <be...@scalytics.io>
AuthorDate: Tue Apr 20 18:17:09 2021 -0400
[WAYANG-28] Add javadoc to identifier package
---
.../plugin/hackit/core/identifiers/HackitID.java | 3 ++
.../hackit/core/identifiers/HackitIDGenerator.java | 43 ++++++++++++++++++++--
.../generator/DistributeSequencial.java | 13 +++++++
.../{TwitterSnowflake.java => SnowflakeID.java} | 39 ++++++++++++++++----
4 files changed, 88 insertions(+), 10 deletions(-)
diff --git a/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/identifiers/HackitID.java b/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/identifiers/HackitID.java
index 1b55679..f21592e 100644
--- a/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/identifiers/HackitID.java
+++ b/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/identifiers/HackitID.java
@@ -17,5 +17,8 @@
*/
package org.apache.wayang.plugin.hackit.core.identifiers;
+/**
+ * TODO Maybe need to be deleted, however, need to be validate it
+ */
public class HackitID {
}
diff --git a/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/identifiers/HackitIDGenerator.java b/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/identifiers/HackitIDGenerator.java
index ad34332..b41eae2 100644
--- a/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/identifiers/HackitIDGenerator.java
+++ b/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/identifiers/HackitIDGenerator.java
@@ -20,31 +20,57 @@ package org.apache.wayang.plugin.hackit.core.identifiers;
import java.net.InetAddress;
import java.net.NetworkInterface;
import java.security.SecureRandom;
+import java.util.ArrayList;
import java.util.Enumeration;
/**
- * Generate the next ID, N depends of the type that is need
+ * HackitIDGenerator is the base for the generation of ID.
+ *
+ * {@link org.apache.wayang.plugin.hackit.core.tuple.header.Header} use the ID as and unique identifier of the tuple
+ *
+ * Type parameters:
+ * <N> – the type of Identifier of the process
+ * <O> - the type of the identifier that is created
* */
public abstract class HackitIDGenerator<N, O> {
+ /**
+ * is_address_calculated indicate if the address of the worker it already calculated,
+ * because the calculation could be costly in time.
+ */
private boolean is_address_calculated = false;
+
+ /**
+ * address_host is the address where the worker is running.
+ */
protected InetAddress address_host;
- /** This is the identifier of the process, task, or machine, depends of the platform
- * but is use for the generators.
+ /**
+ * This is the identifier of the process, task, or machine, depends of the platform
+ * but is use for the generators, normally correspond to an {@link Integer}
* */
protected N identify_process;
+ /**
+ * Empty Constructor
+ */
public HackitIDGenerator(){
this.identify_process = null;
this.address_host = null;
}
+ /**
+ * Constructor with identifier of the process
+ * @param identify_process identifier it help on the generation of unique ID
+ */
public HackitIDGenerator(N identify_process) {
this.identify_process = identify_process;
getAddress();
}
+ /**
+ * Build the address_host from the host information obtained form the context of execution
+ */
protected void getAddress(){
if( ! this.is_address_calculated ){
try {
@@ -55,6 +81,11 @@ public abstract class HackitIDGenerator<N, O> {
}
}
+ /**
+ * create the worker id depending on the network context information
+ *
+ * @return unique number that identifier the worker as unique inside of the cluster
+ */
protected static int createNodeId() {
int nodeId;
try {
@@ -76,5 +107,11 @@ public abstract class HackitIDGenerator<N, O> {
return nodeId;
}
+ /**
+ * Generate an unique ID to every {@link org.apache.wayang.plugin.hackit.core.tuple.HackitTuple}, this could have repetition depending on the
+ * algorithm of generation
+ *
+ * @return ID
+ */
public abstract O generateId();
}
diff --git a/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/identifiers/generator/DistributeSequencial.java b/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/identifiers/generator/DistributeSequencial.java
index f146d65..43865b0 100644
--- a/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/identifiers/generator/DistributeSequencial.java
+++ b/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/identifiers/generator/DistributeSequencial.java
@@ -19,10 +19,23 @@ package org.apache.wayang.plugin.hackit.core.identifiers.generator;
import org.apache.wayang.plugin.hackit.core.identifiers.HackitIDGenerator;
+/**
+ * DistributeSequencial it instance of {@link HackitIDGenerator}.
+ *
+ * DistributeSequencial asume not parallel job running, and help in small debugs, or single node work
+ */
public class DistributeSequencial extends HackitIDGenerator<Integer, Long> {
+ /**
+ * current save the number of elements already generated
+ */
long current = 0;
+ /**
+ * it add 1 to current and assign the previous number to the ID
+ *
+ * @return long that represent the ID, this could have repetition on parallel processing
+ */
@Override
public Long generateId() {
Long tmp = current;
diff --git a/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/identifiers/generator/TwitterSnowflake.java b/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/identifiers/generator/SnowflakeID.java
similarity index 75%
rename from wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/identifiers/generator/TwitterSnowflake.java
rename to wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/identifiers/generator/SnowflakeID.java
index 05d0998..2246915 100644
--- a/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/identifiers/generator/TwitterSnowflake.java
+++ b/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/identifiers/generator/SnowflakeID.java
@@ -21,7 +21,10 @@ import org.apache.wayang.plugin.hackit.core.identifiers.HackitIDGenerator;
import java.time.Instant;
-public class TwitterSnowflake extends HackitIDGenerator<Integer, Long> {
+/**
+ * SnowflackID is and implementation of <a href="https://en.wikipedia.org/wiki/Snowflake_ID">this</a>
+ */
+public class SnowflakeID extends HackitIDGenerator<Integer, Long> {
private static final int TOTAL_BITS = 64;
private static final int EPOCH_BITS = 42;
private static final int NODE_ID_BITS = 10;
@@ -31,21 +34,26 @@ public class TwitterSnowflake extends HackitIDGenerator<Integer, Long> {
private static final int maxSequence = (int)(Math.pow(2, SEQUENCE_BITS) - 1);
// Custom Epoch (January 1, 2015 Midnight UTC = 2015-01-01T00:00:00Z)
+ //TODO: add this element from configuration
private static final long CUSTOM_EPOCH = 1420070400000L;
private volatile long lastTimestamp = -1L;
private volatile long sequence = 0L;
- // Create SequenceGenerator with a nodeId
- public TwitterSnowflake(int nodeId) {
+ /**
+ * Create SequenceGenerator with a nodeId
+ */
+ public SnowflakeID(int nodeId) {
if(nodeId < 0 || nodeId > maxNodeId) {
throw new IllegalArgumentException(String.format("NodeId must be between %d and %d", 0, maxNodeId));
}
this.identify_process = nodeId;
}
- // Let SequenceGenerator generate a nodeId
- public TwitterSnowflake() {
+ /**
+ * Let SequenceGenerator generate a nodeId
+ */
+ public SnowflakeID() {
this( createNodeId() & maxNodeId);
}
@@ -54,6 +62,12 @@ public class TwitterSnowflake extends HackitIDGenerator<Integer, Long> {
return this.nextId();
}
+ /**
+ * Generate the next ID, this method is synchronized because several {@link Thread} could exist
+ * on one unique worker.
+ *
+ * @return the new ID
+ */
public synchronized long nextId() {
long currentTimestamp = timestamp();
@@ -80,12 +94,23 @@ public class TwitterSnowflake extends HackitIDGenerator<Integer, Long> {
return id;
}
- // Get current timestamp in milliseconds, adjust for the custom epoch.
+ /**
+ * Get current timestamp in milliseconds, adjust for the custom epoch.
+ *
+ * @return the Timestamp
+ */
private static long timestamp() {
return Instant.now().toEpochMilli() - CUSTOM_EPOCH;
}
- // Block and wait till next millisecond
+ /**
+ * Block and wait till next millisecond, this is used when the number of elements of one epoch
+ * overflow the max possible number of one epoch
+ *
+ * @param currentTimestamp
+ *
+ * @return the new timestamp after the waiting time
+ */
private long waitNextMillis(long currentTimestamp) {
while (currentTimestamp == lastTimestamp) {
currentTimestamp = timestamp();
[incubator-wayang] 02/05: [WAYANG-28] Add javadoc to iterator
package
Posted by be...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bertty pushed a commit to branch debugger
in repository https://gitbox.apache.org/repos/asf/incubator-wayang.git
commit 6fddce166571e96feec66f91a6c3a3979c8a917b
Author: Bertty Contreras-Rojas <be...@scalytics.io>
AuthorDate: Thu Apr 29 23:10:20 2021 -0400
[WAYANG-28] Add javadoc to iterator package
---
.../hackit/core/iterator/ElementPlusIterator.java | 25 +++++++++++++
.../hackit/core/iterator/FunctionIterator.java | 23 ++++++++++++
.../hackit/core/iterator/HackitIterator.java | 13 ++++++-
.../hackit/core/iterator/OneElementIterator.java | 41 ++++++++++++++++++++++
4 files changed, 101 insertions(+), 1 deletion(-)
diff --git a/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/iterator/ElementPlusIterator.java b/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/iterator/ElementPlusIterator.java
index 32e52c9..e385718 100644
--- a/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/iterator/ElementPlusIterator.java
+++ b/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/iterator/ElementPlusIterator.java
@@ -19,13 +19,38 @@ package org.apache.wayang.plugin.hackit.core.iterator;
import java.util.Iterator;
+/**
+ * ElementPlusIterator extends from {@link Iterator}.
+ *
+ * ElementPlusIterator add one unique element to a know {@link Iterator}, this unique element
+ * it will be consumed first.
+ *
+ * @param <T> type of the element of the iterator
+ */
public class ElementPlusIterator<T> implements Iterator<T> {
+ /**
+ * Indicate if the first element it was consumed or not yet
+ */
private boolean element_consumed = false;
+
+ /**
+ * The element that was included to the iterator
+ */
private T element;
+
+ /**
+ * The iterator where the element was added
+ */
private Iterator<T> iterator;
+ /**
+ * Construct of ElementPlusIterator
+ *
+ * @param element is the object that will be consumed first
+ * @param iterator iterator that will be consumed after the original element
+ */
public ElementPlusIterator(T element, Iterator<T> iterator) {
if(element == null){
throw new RuntimeException("the element can't be null");
diff --git a/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/iterator/FunctionIterator.java b/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/iterator/FunctionIterator.java
index 286566f..5cb22d8 100644
--- a/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/iterator/FunctionIterator.java
+++ b/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/iterator/FunctionIterator.java
@@ -21,11 +21,34 @@ import java.io.Serializable;
import java.util.Iterator;
import java.util.function.Function;
+/**
+ * FunctionIterator implements {@link Iterator} and {@link Serializable}, because the function could be
+ * serializad to be sent to several places at runtime
+ *
+ * FunctionIterator provide the option of convert the data in the iterator using an {@link Function},
+ * this will transform the element to the new kind
+ *
+ * @param <I> Type before of the transformation in the iterator
+ * @param <O> Type after the transformation, this could be the same to <I>
+ */
public class FunctionIterator<I, O> implements Iterator<O>, Serializable {
+ /**
+ * base is an {@link Iterator} that will be transformed during runtime
+ */
private Iterator<I> base;
+
+ /**
+ * function is an {@link Function} that will convert the element inside of <code>base</code>
+ */
private Function<I, O> function;
+ /**
+ * Construct of FunctionIterator
+ *
+ * @param base {@link Iterator} that will be transformed at the consumed time
+ * @param function {@link Function} that will convert the data inside of <code>base</code>
+ */
public FunctionIterator(Iterator<I> base, Function<I, O> function) {
this.base = base;
this.function = function;
diff --git a/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/iterator/HackitIterator.java b/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/iterator/HackitIterator.java
index 3ce2f84..42166d2 100644
--- a/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/iterator/HackitIterator.java
+++ b/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/iterator/HackitIterator.java
@@ -22,9 +22,20 @@ import org.apache.wayang.plugin.hackit.core.tuple.HackitTuple;
import java.util.Iterator;
import java.util.function.Function;
-
+/**
+ * HackitIterator extends {@link FunctionIterator} and provides an signature for the conversion of
+ * the tuples into a {@link HackitTuple} during they process.
+ *
+ * @param <K> type of the key on the {@link HackitTuple}
+ * @param <T> type of the element that it contains the {@link HackitTuple}
+ */
public class HackitIterator<K, T> extends FunctionIterator<T, HackitTuple<K, T>>{
+ /**
+ * default construct
+ * @param base {@link Iterator} that element will be wrapper inside of a {@link HackitTuple}
+ * @param function it a {@link Function} that convert the the {@link HackitTuple}
+ */
public HackitIterator(Iterator<T> base, Function<T, HackitTuple<K, T>> function) {
super(base, function);
}
diff --git a/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/iterator/OneElementIterator.java b/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/iterator/OneElementIterator.java
index 1ebf412..811413e 100644
--- a/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/iterator/OneElementIterator.java
+++ b/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/iterator/OneElementIterator.java
@@ -20,17 +20,51 @@ package org.apache.wayang.plugin.hackit.core.iterator;
import java.util.Iterator;
import java.util.NoSuchElementException;
+/**
+ * OneElementIterator implements {@link Iterator} and {@link Iterable}
+ *
+ * OneElementIterator will contains just one element, but that element need to be treated as
+ * Iterator, this class allow handler all the functionality that are related to that.
+ *
+ * @param <T> type of the element
+ */
public class OneElementIterator<T> implements Iterator<T>, Iterable<T>{
+ /**
+ * indicate if the element could be removed
+ */
private final boolean removeAllowed;
+
+ /**
+ * Indicate if the process is started or not
+ */
private boolean beforeFirst;
+
+ /**
+ * allows to follow the process as the object didn't exist
+ */
private boolean removed;
+
+ /**
+ * is the element that it will "process" at the iteration time
+ */
private T object;
+ /**
+ * Default Construct
+ *
+ * @param object element that will be process
+ */
public OneElementIterator(T object) {
this(object, true);
}
+ /**
+ * Construct that allows indicate the option of remove the object
+ *
+ * @param object element that will be process
+ * @param removeAllowed indicate if the element could be removed
+ */
public OneElementIterator(T object, boolean removeAllowed) {
this.beforeFirst = true;
this.removed = false;
@@ -38,10 +72,12 @@ public class OneElementIterator<T> implements Iterator<T>, Iterable<T>{
this.removeAllowed = removeAllowed;
}
+ @Override
public boolean hasNext() {
return this.beforeFirst && !this.removed;
}
+ @Override
public T next() {
if (this.beforeFirst && !this.removed) {
this.beforeFirst = false;
@@ -51,6 +87,7 @@ public class OneElementIterator<T> implements Iterator<T>, Iterable<T>{
}
}
+ @Override
public void remove() {
if (this.removeAllowed) {
if (!this.removed && !this.beforeFirst) {
@@ -64,6 +101,10 @@ public class OneElementIterator<T> implements Iterator<T>, Iterable<T>{
}
}
+ /**
+ * reset the Iterator if is needed, because the iterator it just one element, then is possible to do several
+ * full iteration on top of the element.
+ */
public void reset() {
this.beforeFirst = true;
}
[incubator-wayang] 04/05: [WAYANG-28] rename
wayang-hackit-shipper-rabbitmq to wayang-hackit-shipper-kafka
Posted by be...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bertty pushed a commit to branch debugger
in repository https://gitbox.apache.org/repos/asf/incubator-wayang.git
commit 3a56f9be7005bd4f0ceb49510fe62bce1241b797
Author: Bertty Contreras-Rojas <be...@scalytics.io>
AuthorDate: Tue May 4 10:58:21 2021 -0400
[WAYANG-28] rename wayang-hackit-shipper-rabbitmq to wayang-hackit-shipper-kafka
---
wayang-plugins/wayang-hackit/wayang-hackit-shipper/pom.xml | 2 +-
.../pom.xml | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/wayang-plugins/wayang-hackit/wayang-hackit-shipper/pom.xml b/wayang-plugins/wayang-hackit/wayang-hackit-shipper/pom.xml
index 6f0c290..c1e128f 100644
--- a/wayang-plugins/wayang-hackit/wayang-hackit-shipper/pom.xml
+++ b/wayang-plugins/wayang-hackit/wayang-hackit-shipper/pom.xml
@@ -30,7 +30,7 @@
<artifactId>wayang-hackit-shipper</artifactId>
<packaging>pom</packaging>
<modules>
- <module>wayang-hackit-shipper-rabbitmq</module>
+ <module>wayang-hackit-shipper-kafka</module>
</modules>
<properties>
diff --git a/wayang-plugins/wayang-hackit/wayang-hackit-shipper/wayang-hackit-shipper-rabbitmq/pom.xml b/wayang-plugins/wayang-hackit/wayang-hackit-shipper/wayang-hackit-shipper-kafka/pom.xml
similarity index 96%
rename from wayang-plugins/wayang-hackit/wayang-hackit-shipper/wayang-hackit-shipper-rabbitmq/pom.xml
rename to wayang-plugins/wayang-hackit/wayang-hackit-shipper/wayang-hackit-shipper-kafka/pom.xml
index 36fff1c..83fe74d 100644
--- a/wayang-plugins/wayang-hackit/wayang-hackit-shipper/wayang-hackit-shipper-rabbitmq/pom.xml
+++ b/wayang-plugins/wayang-hackit/wayang-hackit-shipper/wayang-hackit-shipper-kafka/pom.xml
@@ -27,7 +27,7 @@
</parent>
<modelVersion>4.0.0</modelVersion>
- <artifactId>wayang-hackit-shipper-rabbitmq</artifactId>
+ <artifactId>wayang-hackit-shipper-kafka</artifactId>
<properties>
<maven.compiler.source>8</maven.compiler.source>