You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by fh...@apache.org on 2015/12/15 21:36:22 UTC
[1/2] flink git commit: [FLINK-2882] [core] Improve performance of
string conversions - Memoize string representations of AbstractID. - Use
lookup table for byte-to-hex conversion in StringUtils.
Repository: flink
Updated Branches:
refs/heads/master e24861886 -> 19cbc787b
[FLINK-2882] [core] Improve performance of string conversions
- Memoize string representations of AbstractID.
- Use lookup table for byte-to-hex conversion in StringUtils.
This closes #1455
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/b9ce8abb
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/b9ce8abb
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/b9ce8abb
Branch: refs/heads/master
Commit: b9ce8abb1bfc0cbc43e8c1800b4aeb7aee7c6e62
Parents: e248618
Author: Greg Hogan <co...@greghogan.com>
Authored: Mon Dec 14 14:47:33 2015 -0500
Committer: Fabian Hueske <fh...@apache.org>
Committed: Tue Dec 15 21:14:33 2015 +0100
----------------------------------------------------------------------
.../java/org/apache/flink/util/AbstractID.java | 33 +++++++++++++++-----
.../java/org/apache/flink/util/StringUtils.java | 16 +++++++---
2 files changed, 38 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/b9ce8abb/flink-core/src/main/java/org/apache/flink/util/AbstractID.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/util/AbstractID.java b/flink-core/src/main/java/org/apache/flink/util/AbstractID.java
index 250869f..5a57900 100644
--- a/flink-core/src/main/java/org/apache/flink/util/AbstractID.java
+++ b/flink-core/src/main/java/org/apache/flink/util/AbstractID.java
@@ -48,6 +48,12 @@ public class AbstractID implements IOReadableWritable, Comparable<AbstractID>, j
/** The lower part of the actual ID */
protected long lowerPart;
+ /** The memoized value returned by toString() */
+ private String toString;
+
+ /** The memoized value returned by toShortString() */
+ private String toShortString;
+
// --------------------------------------------------------------------------------------------
/**
@@ -137,6 +143,9 @@ public class AbstractID implements IOReadableWritable, Comparable<AbstractID>, j
public void read(DataInputView in) throws IOException {
this.lowerPart = in.readLong();
this.upperPart = in.readLong();
+
+ this.toString = null;
+ this.toShortString = null;
}
@Override
@@ -169,16 +178,26 @@ public class AbstractID implements IOReadableWritable, Comparable<AbstractID>, j
@Override
public String toString() {
- final byte[] ba = new byte[SIZE];
- longToByteArray(this.lowerPart, ba, 0);
- longToByteArray(this.upperPart, ba, SIZE_OF_LONG);
- return StringUtils.byteToHexString(ba);
+ if (this.toString == null) {
+ final byte[] ba = new byte[SIZE];
+ longToByteArray(this.lowerPart, ba, 0);
+ longToByteArray(this.upperPart, ba, SIZE_OF_LONG);
+
+ this.toString = StringUtils.byteToHexString(ba);
+ }
+
+ return this.toString;
}
public String toShortString() {
- final byte[] ba = new byte[SIZE_OF_LONG];
- longToByteArray(upperPart, ba, 0);
- return StringUtils.byteToHexString(ba);
+ if (this.toShortString == null) {
+ final byte[] ba = new byte[SIZE_OF_LONG];
+ longToByteArray(upperPart, ba, 0);
+
+ this.toShortString = StringUtils.byteToHexString(ba);
+ }
+
+ return this.toShortString;
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/b9ce8abb/flink-core/src/main/java/org/apache/flink/util/StringUtils.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/util/StringUtils.java b/flink-core/src/main/java/org/apache/flink/util/StringUtils.java
index 9c7bd8b..2257183 100644
--- a/flink-core/src/main/java/org/apache/flink/util/StringUtils.java
+++ b/flink-core/src/main/java/org/apache/flink/util/StringUtils.java
@@ -58,17 +58,25 @@ public final class StringUtils {
* @param end
* end index, exclusively
* @return hex string representation of the byte array
+ *
+ * @see org.apache.commons.codec.binary.Hex#encodeHexString(byte[])
*/
public static String byteToHexString(final byte[] bytes, final int start, final int end) {
if (bytes == null) {
throw new IllegalArgumentException("bytes == null");
}
- final StringBuilder s = new StringBuilder();
- for (int i = start; i < end; i++) {
- s.append(String.format("%02x", bytes[i]));
+ final char[] HEX_CHARS = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' };
+
+ int length = end - start;
+ char[] out = new char[length * 2];
+
+ for (int i = start, j = 0; i < end; i++) {
+ out[j++] = HEX_CHARS[(0xF0 & bytes[i]) >>> 4];
+ out[j++] = HEX_CHARS[0x0F & bytes[i]];
}
- return s.toString();
+
+ return new String(out);
}
/**
[2/2] flink git commit: [FLINK-3147] Expose HadoopOutputFormatBase
fields as protected
Posted by fh...@apache.org.
[FLINK-3147] Expose HadoopOutputFormatBase fields as protected
This closes #1442
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/19cbc787
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/19cbc787
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/19cbc787
Branch: refs/heads/master
Commit: 19cbc787b9f53dd494d77899e00111f437e907b6
Parents: b9ce8ab
Author: Nick Dimiduk <nd...@apache.org>
Authored: Tue Dec 8 10:20:58 2015 -0800
Committer: Fabian Hueske <fh...@apache.org>
Committed: Tue Dec 15 21:15:34 2015 +0100
----------------------------------------------------------------------
.../java/hadoop/mapred/HadoopOutputFormatBase.java | 14 +++++++-------
.../hadoop/mapreduce/HadoopOutputFormatBase.java | 16 ++++++++--------
2 files changed, 15 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/19cbc787/flink-java/src/main/java/org/apache/flink/api/java/hadoop/mapred/HadoopOutputFormatBase.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/hadoop/mapred/HadoopOutputFormatBase.java b/flink-java/src/main/java/org/apache/flink/api/java/hadoop/mapred/HadoopOutputFormatBase.java
index 40214f2..eef0905 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/hadoop/mapred/HadoopOutputFormatBase.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/hadoop/mapred/HadoopOutputFormatBase.java
@@ -58,15 +58,15 @@ public abstract class HadoopOutputFormatBase<K, V, T> extends HadoopOutputFormat
// Hadoop parallelizes tasks across JVMs which is why they might rely on this JVM isolation.
// In contrast, Flink parallelizes using Threads, so multiple Hadoop OutputFormat instances
// might be used in the same JVM.
- private static final Object OPEN_MUTEX = new Object();
- private static final Object CONFIGURE_MUTEX = new Object();
- private static final Object CLOSE_MUTEX = new Object();
+ protected static final Object OPEN_MUTEX = new Object();
+ protected static final Object CONFIGURE_MUTEX = new Object();
+ protected static final Object CLOSE_MUTEX = new Object();
- private JobConf jobConf;
- private org.apache.hadoop.mapred.OutputFormat<K,V> mapredOutputFormat;
+ protected JobConf jobConf;
+ protected org.apache.hadoop.mapred.OutputFormat<K,V> mapredOutputFormat;
protected transient RecordWriter<K,V> recordWriter;
- private transient OutputCommitter outputCommitter;
- private transient TaskAttemptContext context;
+ protected transient OutputCommitter outputCommitter;
+ protected transient TaskAttemptContext context;
public HadoopOutputFormatBase(org.apache.hadoop.mapred.OutputFormat<K, V> mapredOutputFormat, JobConf job) {
super(job.getCredentials());
http://git-wip-us.apache.org/repos/asf/flink/blob/19cbc787/flink-java/src/main/java/org/apache/flink/api/java/hadoop/mapreduce/HadoopOutputFormatBase.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/hadoop/mapreduce/HadoopOutputFormatBase.java b/flink-java/src/main/java/org/apache/flink/api/java/hadoop/mapreduce/HadoopOutputFormatBase.java
index dc475e8..d2bb310 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/hadoop/mapreduce/HadoopOutputFormatBase.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/hadoop/mapreduce/HadoopOutputFormatBase.java
@@ -53,16 +53,16 @@ public abstract class HadoopOutputFormatBase<K, V, T> extends HadoopOutputFormat
// Hadoop parallelizes tasks across JVMs which is why they might rely on this JVM isolation.
// In contrast, Flink parallelizes using Threads, so multiple Hadoop OutputFormat instances
// might be used in the same JVM.
- private static final Object OPEN_MUTEX = new Object();
- private static final Object CONFIGURE_MUTEX = new Object();
- private static final Object CLOSE_MUTEX = new Object();
+ protected static final Object OPEN_MUTEX = new Object();
+ protected static final Object CONFIGURE_MUTEX = new Object();
+ protected static final Object CLOSE_MUTEX = new Object();
- private org.apache.hadoop.conf.Configuration configuration;
- private org.apache.hadoop.mapreduce.OutputFormat<K,V> mapreduceOutputFormat;
+ protected org.apache.hadoop.conf.Configuration configuration;
+ protected org.apache.hadoop.mapreduce.OutputFormat<K,V> mapreduceOutputFormat;
protected transient RecordWriter<K,V> recordWriter;
- private transient OutputCommitter outputCommitter;
- private transient TaskAttemptContext context;
- private transient int taskNumber;
+ protected transient OutputCommitter outputCommitter;
+ protected transient TaskAttemptContext context;
+ protected transient int taskNumber;
public HadoopOutputFormatBase(org.apache.hadoop.mapreduce.OutputFormat<K, V> mapreduceOutputFormat, Job job) {
super(job.getCredentials());