You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@wayang.apache.org by be...@apache.org on 2021/05/04 15:00:58 UTC
[incubator-wayang] 01/05: [WAYANG-28] Add javadoc to identifier
package
This is an automated email from the ASF dual-hosted git repository.
bertty pushed a commit to branch debugger
in repository https://gitbox.apache.org/repos/asf/incubator-wayang.git
commit 1a88f89324b1581f9475a57598d26c6d667405d9
Author: Bertty Contreras-Rojas <be...@scalytics.io>
AuthorDate: Tue Apr 20 18:17:09 2021 -0400
[WAYANG-28] Add javadoc to identifier package
---
.../plugin/hackit/core/identifiers/HackitID.java | 3 ++
.../hackit/core/identifiers/HackitIDGenerator.java | 43 ++++++++++++++++++++--
.../generator/DistributeSequencial.java | 13 +++++++
.../{TwitterSnowflake.java => SnowflakeID.java} | 39 ++++++++++++++++----
4 files changed, 88 insertions(+), 10 deletions(-)
diff --git a/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/identifiers/HackitID.java b/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/identifiers/HackitID.java
index 1b55679..f21592e 100644
--- a/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/identifiers/HackitID.java
+++ b/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/identifiers/HackitID.java
@@ -17,5 +17,8 @@
*/
package org.apache.wayang.plugin.hackit.core.identifiers;
+/**
+ * TODO Maybe need to be deleted, however, need to be validate it
+ */
public class HackitID {
}
diff --git a/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/identifiers/HackitIDGenerator.java b/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/identifiers/HackitIDGenerator.java
index ad34332..b41eae2 100644
--- a/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/identifiers/HackitIDGenerator.java
+++ b/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/identifiers/HackitIDGenerator.java
@@ -20,31 +20,57 @@ package org.apache.wayang.plugin.hackit.core.identifiers;
import java.net.InetAddress;
import java.net.NetworkInterface;
import java.security.SecureRandom;
+import java.util.ArrayList;
import java.util.Enumeration;
/**
- * Generate the next ID, N depends of the type that is need
+ * HackitIDGenerator is the base for the generation of ID.
+ *
+ * {@link org.apache.wayang.plugin.hackit.core.tuple.header.Header} use the ID as and unique identifier of the tuple
+ *
+ * Type parameters:
+ * <N> – the type of Identifier of the process
+ * <O> - the type of the identifier that is created
* */
public abstract class HackitIDGenerator<N, O> {
+ /**
+ * is_address_calculated indicate if the address of the worker it already calculated,
+ * because the calculation could be costly in time.
+ */
private boolean is_address_calculated = false;
+
+ /**
+ * address_host is the address where the worker is running.
+ */
protected InetAddress address_host;
- /** This is the identifier of the process, task, or machine, depends of the platform
- * but is use for the generators.
+ /**
+ * This is the identifier of the process, task, or machine, depends of the platform
+ * but is use for the generators, normally correspond to an {@link Integer}
* */
protected N identify_process;
+ /**
+ * Empty Constructor
+ */
public HackitIDGenerator(){
this.identify_process = null;
this.address_host = null;
}
+ /**
+ * Constructor with identifier of the process
+ * @param identify_process identifier it help on the generation of unique ID
+ */
public HackitIDGenerator(N identify_process) {
this.identify_process = identify_process;
getAddress();
}
+ /**
+ * Build the address_host from the host information obtained form the context of execution
+ */
protected void getAddress(){
if( ! this.is_address_calculated ){
try {
@@ -55,6 +81,11 @@ public abstract class HackitIDGenerator<N, O> {
}
}
+ /**
+ * create the worker id depending on the network context information
+ *
+ * @return unique number that identifier the worker as unique inside of the cluster
+ */
protected static int createNodeId() {
int nodeId;
try {
@@ -76,5 +107,11 @@ public abstract class HackitIDGenerator<N, O> {
return nodeId;
}
+ /**
+ * Generate an unique ID to every {@link org.apache.wayang.plugin.hackit.core.tuple.HackitTuple}, this could have repetition depending on the
+ * algorithm of generation
+ *
+ * @return ID
+ */
public abstract O generateId();
}
diff --git a/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/identifiers/generator/DistributeSequencial.java b/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/identifiers/generator/DistributeSequencial.java
index f146d65..43865b0 100644
--- a/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/identifiers/generator/DistributeSequencial.java
+++ b/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/identifiers/generator/DistributeSequencial.java
@@ -19,10 +19,23 @@ package org.apache.wayang.plugin.hackit.core.identifiers.generator;
import org.apache.wayang.plugin.hackit.core.identifiers.HackitIDGenerator;
+/**
+ * DistributeSequencial it instance of {@link HackitIDGenerator}.
+ *
+ * DistributeSequencial asume not parallel job running, and help in small debugs, or single node work
+ */
public class DistributeSequencial extends HackitIDGenerator<Integer, Long> {
+ /**
+ * current save the number of elements already generated
+ */
long current = 0;
+ /**
+ * it add 1 to current and assign the previous number to the ID
+ *
+ * @return long that represent the ID, this could have repetition on parallel processing
+ */
@Override
public Long generateId() {
Long tmp = current;
diff --git a/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/identifiers/generator/TwitterSnowflake.java b/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/identifiers/generator/SnowflakeID.java
similarity index 75%
rename from wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/identifiers/generator/TwitterSnowflake.java
rename to wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/identifiers/generator/SnowflakeID.java
index 05d0998..2246915 100644
--- a/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/identifiers/generator/TwitterSnowflake.java
+++ b/wayang-plugins/wayang-hackit/wayang-hackit-core/src/main/java/org/apache/wayang/plugin/hackit/core/identifiers/generator/SnowflakeID.java
@@ -21,7 +21,10 @@ import org.apache.wayang.plugin.hackit.core.identifiers.HackitIDGenerator;
import java.time.Instant;
-public class TwitterSnowflake extends HackitIDGenerator<Integer, Long> {
+/**
+ * SnowflackID is and implementation of <a href="https://en.wikipedia.org/wiki/Snowflake_ID">this</a>
+ */
+public class SnowflakeID extends HackitIDGenerator<Integer, Long> {
private static final int TOTAL_BITS = 64;
private static final int EPOCH_BITS = 42;
private static final int NODE_ID_BITS = 10;
@@ -31,21 +34,26 @@ public class TwitterSnowflake extends HackitIDGenerator<Integer, Long> {
private static final int maxSequence = (int)(Math.pow(2, SEQUENCE_BITS) - 1);
// Custom Epoch (January 1, 2015 Midnight UTC = 2015-01-01T00:00:00Z)
+ //TODO: add this element from configuration
private static final long CUSTOM_EPOCH = 1420070400000L;
private volatile long lastTimestamp = -1L;
private volatile long sequence = 0L;
- // Create SequenceGenerator with a nodeId
- public TwitterSnowflake(int nodeId) {
+ /**
+ * Create SequenceGenerator with a nodeId
+ */
+ public SnowflakeID(int nodeId) {
if(nodeId < 0 || nodeId > maxNodeId) {
throw new IllegalArgumentException(String.format("NodeId must be between %d and %d", 0, maxNodeId));
}
this.identify_process = nodeId;
}
- // Let SequenceGenerator generate a nodeId
- public TwitterSnowflake() {
+ /**
+ * Let SequenceGenerator generate a nodeId
+ */
+ public SnowflakeID() {
this( createNodeId() & maxNodeId);
}
@@ -54,6 +62,12 @@ public class TwitterSnowflake extends HackitIDGenerator<Integer, Long> {
return this.nextId();
}
+ /**
+ * Generate the next ID, this method is synchronized because several {@link Thread} could exist
+ * on one unique worker.
+ *
+ * @return the new ID
+ */
public synchronized long nextId() {
long currentTimestamp = timestamp();
@@ -80,12 +94,23 @@ public class TwitterSnowflake extends HackitIDGenerator<Integer, Long> {
return id;
}
- // Get current timestamp in milliseconds, adjust for the custom epoch.
+ /**
+ * Get current timestamp in milliseconds, adjust for the custom epoch.
+ *
+ * @return the Timestamp
+ */
private static long timestamp() {
return Instant.now().toEpochMilli() - CUSTOM_EPOCH;
}
- // Block and wait till next millisecond
+ /**
+ * Block and wait till next millisecond, this is used when the number of elements of one epoch
+ * overflow the max possible number of one epoch
+ *
+ * @param currentTimestamp
+ *
+ * @return the new timestamp after the waiting time
+ */
private long waitNextMillis(long currentTimestamp) {
while (currentTimestamp == lastTimestamp) {
currentTimestamp = timestamp();