You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by gr...@apache.org on 2017/12/22 19:12:24 UTC
[1/5] flink git commit: [FLINK-8223] [build] Update Hadoop versions
Repository: flink
Updated Branches:
refs/heads/master 1a98e327e -> 15a0dc4ae
[FLINK-8223] [build] Update Hadoop versions
Update 2.7.3 to 2.7.5 and 2.8.0 to 2.8.3
This closes #5137
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/d3cd51a3
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/d3cd51a3
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/d3cd51a3
Branch: refs/heads/master
Commit: d3cd51a3f9fbb3ffbe6d23a57ff3884733eb47fa
Parents: 8987de3
Author: Greg Hogan <co...@greghogan.com>
Authored: Thu Dec 7 13:29:29 2017 -0500
Committer: Greg Hogan <co...@greghogan.com>
Committed: Thu Dec 21 10:35:53 2017 -0500
----------------------------------------------------------------------
.travis.yml | 10 +++++-----
tools/releasing/create_binary_release.sh | 4 ++--
2 files changed, 7 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/d3cd51a3/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index 5e2ef74..f6a2ff6 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -39,27 +39,27 @@ matrix:
- jdk: "oraclejdk8"
env:
- TEST="core"
- - PROFILE="-Dhadoop.version=2.8.0"
+ - PROFILE="-Dhadoop.version=2.8.3"
- CACHE_NAME=JDK8_H280_CO
- jdk: "oraclejdk8"
env:
- TEST="libraries"
- - PROFILE="-Dhadoop.version=2.8.0"
+ - PROFILE="-Dhadoop.version=2.8.3"
- CACHE_NAME=JDK8_H280_L
- jdk: "oraclejdk8"
env:
- TEST="connectors"
- - PROFILE="-Dhadoop.version=2.8.0 -Pinclude-kinesis"
+ - PROFILE="-Dhadoop.version=2.8.3 -Pinclude-kinesis"
- CACHE_NAME=JDK8_H280_CN
- jdk: "oraclejdk8"
env:
- TEST="tests"
- - PROFILE="-Dhadoop.version=2.8.0"
+ - PROFILE="-Dhadoop.version=2.8.3"
- CACHE_NAME=JDK8_H280_T
- jdk: "oraclejdk8"
env:
- TEST="misc"
- - PROFILE="-Dhadoop.version=2.8.0 -Dinclude_hadoop_aws"
+ - PROFILE="-Dhadoop.version=2.8.3 -Dinclude_hadoop_aws"
- CACHE_NAME=JDK8_H280_M
- jdk: "openjdk8"
env:
http://git-wip-us.apache.org/repos/asf/flink/blob/d3cd51a3/tools/releasing/create_binary_release.sh
----------------------------------------------------------------------
diff --git a/tools/releasing/create_binary_release.sh b/tools/releasing/create_binary_release.sh
index 82b98a0..24d2294 100755
--- a/tools/releasing/create_binary_release.sh
+++ b/tools/releasing/create_binary_release.sh
@@ -85,8 +85,8 @@ if [ "$SCALA_VERSION" == "none" ] && [ "$HADOOP_VERSION" == "none" ]; then
make_binary_release "" "-DwithoutHadoop" "2.11"
make_binary_release "hadoop24" "-Dhadoop.version=2.4.1" "2.11"
make_binary_release "hadoop26" "-Dhadoop.version=2.6.5" "2.11"
- make_binary_release "hadoop27" "-Dhadoop.version=2.7.3" "2.11"
- make_binary_release "hadoop28" "-Dhadoop.version=2.8.0" "2.11"
+ make_binary_release "hadoop27" "-Dhadoop.version=2.7.5" "2.11"
+ make_binary_release "hadoop28" "-Dhadoop.version=2.8.3" "2.11"
elif [ "$SCALA_VERSION" == none ] && [ "$HADOOP_VERSION" != "none" ]
then
make_binary_release "hadoop2" "-Dhadoop.version=$HADOOP_VERSION" "2.11"
[5/5] flink git commit: [hotfix] [docs] Fix typos in MemorySegment
class
Posted by gr...@apache.org.
[hotfix] [docs] Fix typos in MemorySegment class
This closes #5199
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/15a0dc4a
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/15a0dc4a
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/15a0dc4a
Branch: refs/heads/master
Commit: 15a0dc4aecea4a5cff26abef00a067577a27c985
Parents: cc8f70b
Author: Cristian <me...@cristian.io>
Authored: Thu Dec 21 05:33:18 2017 -0800
Committer: Greg Hogan <co...@greghogan.com>
Committed: Thu Dec 21 11:47:05 2017 -0500
----------------------------------------------------------------------
.../apache/flink/core/memory/MemorySegment.java | 18 +++++++++---------
1 file changed, 9 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/15a0dc4a/flink-core/src/main/java/org/apache/flink/core/memory/MemorySegment.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/core/memory/MemorySegment.java b/flink-core/src/main/java/org/apache/flink/core/memory/MemorySegment.java
index 3ad1c63..44b5eca 100644
--- a/flink-core/src/main/java/org/apache/flink/core/memory/MemorySegment.java
+++ b/flink-core/src/main/java/org/apache/flink/core/memory/MemorySegment.java
@@ -272,7 +272,7 @@ public abstract class MemorySegment {
// - Grab mutable fields onto the stack before checking and using. This
// guards us against concurrent modifications which invalidate the
// pointers
- // - Use subtrations for range checks, as they are tolerant
+ // - Use subtractions for range checks, as they are tolerant
//------------------------------------------------------------------------
/**
@@ -402,7 +402,7 @@ public abstract class MemorySegment {
}
/**
- * Reads an character value (16 bit, 2 bytes) from the given position, in little-endian byte order.
+ * Reads a character value (16 bit, 2 bytes) from the given position, in little-endian byte order.
* This method's speed depends on the system's native byte order, and it
* is possibly slower than {@link #getChar(int)}. For most cases (such as
* transient storage in memory or serialization for I/O and network),
@@ -423,7 +423,7 @@ public abstract class MemorySegment {
}
/**
- * Reads an character value (16 bit, 2 bytes) from the given position, in big-endian byte order.
+ * Reads a character value (16 bit, 2 bytes) from the given position, in big-endian byte order.
* This method's speed depends on the system's native byte order, and it
* is possibly slower than {@link #getChar(int)}. For most cases (such as
* transient storage in memory or serialization for I/O and network),
@@ -444,7 +444,7 @@ public abstract class MemorySegment {
}
/**
- * Writes a char value to teh given position, in the system's native byte order.
+ * Writes a char value to the given position, in the system's native byte order.
*
* @param index The position at which the memory will be written.
* @param value The char value to be written.
@@ -476,7 +476,7 @@ public abstract class MemorySegment {
* one in which it is read, and {@link #putChar(int, char)} is the preferable choice.
*
* @param index The position at which the value will be written.
- * @param value The short value to be written.
+ * @param value The char value to be written.
*
* @throws IndexOutOfBoundsException Thrown, if the index is negative, or larger then the segment size minus 2.
*/
@@ -497,7 +497,7 @@ public abstract class MemorySegment {
* one in which it is read, and {@link #putChar(int, char)} is the preferable choice.
*
* @param index The position at which the value will be written.
- * @param value The short value to be written.
+ * @param value The char value to be written.
*
* @throws IndexOutOfBoundsException Thrown, if the index is negative, or larger then the segment size minus 2.
*/
@@ -510,7 +510,7 @@ public abstract class MemorySegment {
}
/**
- * Reads two memory at the given position, composing them into a short value
+ * Reads a short integer value (16 bit, 2 bytes) from the given position, composing them into a short value
* according to the current byte order.
*
* @param index The position from which the memory will be read.
@@ -534,7 +534,7 @@ public abstract class MemorySegment {
}
/**
- * Reads an short integer value (16 bit, 2 bytes) from the given position, in little-endian byte order.
+ * Reads a short integer value (16 bit, 2 bytes) from the given position, in little-endian byte order.
* This method's speed depends on the system's native byte order, and it
* is possibly slower than {@link #getShort(int)}. For most cases (such as
* transient storage in memory or serialization for I/O and network),
@@ -555,7 +555,7 @@ public abstract class MemorySegment {
}
/**
- * Reads an short integer value (16 bit, 2 bytes) from the given position, in big-endian byte order.
+ * Reads a short integer value (16 bit, 2 bytes) from the given position, in big-endian byte order.
* This method's speed depends on the system's native byte order, and it
* is possibly slower than {@link #getShort(int)}. For most cases (such as
* transient storage in memory or serialization for I/O and network),
[4/5] flink git commit: [hotfix] [build] Always include Kafka 0.11
connector
Posted by gr...@apache.org.
[hotfix] [build] Always include Kafka 0.11 connector
Now that Flink only supports builds for Scala 2.11+ we can
unconditionally enable the Kafka 0.11 connector.
This closes #5195
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/cc8f70b1
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/cc8f70b1
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/cc8f70b1
Branch: refs/heads/master
Commit: cc8f70b17aec29f9d92dd537402b4d1c15ad10a7
Parents: d3cd51a
Author: Greg Hogan <co...@greghogan.com>
Authored: Wed Dec 20 12:11:00 2017 -0500
Committer: Greg Hogan <co...@greghogan.com>
Committed: Thu Dec 21 11:46:27 2017 -0500
----------------------------------------------------------------------
flink-connectors/pom.xml | 13 +------------
tools/travis_mvn_watchdog.sh | 5 +----
2 files changed, 2 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/cc8f70b1/flink-connectors/pom.xml
----------------------------------------------------------------------
diff --git a/flink-connectors/pom.xml b/flink-connectors/pom.xml
index e8dd7a8..6f048ca 100644
--- a/flink-connectors/pom.xml
+++ b/flink-connectors/pom.xml
@@ -45,6 +45,7 @@ under the License.
<module>flink-connector-kafka-0.8</module>
<module>flink-connector-kafka-0.9</module>
<module>flink-connector-kafka-0.10</module>
+ <module>flink-connector-kafka-0.11</module>
<module>flink-connector-elasticsearch-base</module>
<module>flink-connector-elasticsearch</module>
<module>flink-connector-elasticsearch2</module>
@@ -75,18 +76,6 @@ under the License.
<!-- See main pom.xml for explanation of profiles -->
<profiles>
- <!-- Kafka 0.11 does not support scala 2.10-->
- <profile>
- <id>scala-2.11</id>
- <activation>
- <property>
- <name>!scala-2.10</name>
- </property>
- </activation>
- <modules>
- <module>flink-connector-kafka-0.11</module>
- </modules>
- </profile>
<!--
We include the kinesis module only optionally because it contains a dependency
licenced under the "Amazon Software License".
http://git-wip-us.apache.org/repos/asf/flink/blob/cc8f70b1/tools/travis_mvn_watchdog.sh
----------------------------------------------------------------------
diff --git a/tools/travis_mvn_watchdog.sh b/tools/travis_mvn_watchdog.sh
index 42f7ae3..036ad29 100755
--- a/tools/travis_mvn_watchdog.sh
+++ b/tools/travis_mvn_watchdog.sh
@@ -91,6 +91,7 @@ flink-connectors/flink-connector-filesystem,\
flink-connectors/flink-connector-kafka-0.8,\
flink-connectors/flink-connector-kafka-0.9,\
flink-connectors/flink-connector-kafka-0.10,\
+flink-connectors/flink-connector-kafka-0.11,\
flink-connectors/flink-connector-kafka-base,\
flink-connectors/flink-connector-nifi,\
flink-connectors/flink-connector-rabbitmq,\
@@ -99,10 +100,6 @@ flink-connectors/flink-connector-twitter"
MODULES_TESTS="\
flink-tests"
-if [[ $PROFILE != *"scala-2.10"* ]]; then
- MODULES_CONNECTORS="$MODULES_CONNECTORS,flink-connectors/flink-connector-kafka-0.11"
-fi
-
if [[ $PROFILE == *"include-kinesis"* ]]; then
case $TEST in
(connectors)
[2/5] flink git commit: [FLINK-8222] [build] Update Scala version
Posted by gr...@apache.org.
[FLINK-8222] [build] Update Scala version
This is an incremental upgrade to the Scala security release 2.11.12.
This closes #5136
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/8987de3b
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/8987de3b
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/8987de3b
Branch: refs/heads/master
Commit: 8987de3b241d23bbcc6ca5640e3cb77972a60be4
Parents: a355df6
Author: Greg Hogan <co...@greghogan.com>
Authored: Thu Dec 7 13:22:00 2017 -0500
Committer: Greg Hogan <co...@greghogan.com>
Committed: Thu Dec 21 10:35:53 2017 -0500
----------------------------------------------------------------------
.../src/main/resources/archetype-resources/pom.xml | 2 +-
pom.xml | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/8987de3b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml
----------------------------------------------------------------------
diff --git a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml
index ca41014..c82c385 100644
--- a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml
+++ b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml
@@ -49,7 +49,7 @@ under the License.
<slf4j.version>@slf4j.version@</slf4j.version>
<log4j.version>@log4j.version@</log4j.version>
<scala.binary.version>2.11</scala.binary.version>
- <scala.version>2.11.11</scala.version>
+ <scala.version>2.11.12</scala.version>
</properties>
<!--
http://git-wip-us.apache.org/repos/asf/flink/blob/8987de3b/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index ec17818..40c7fa5 100644
--- a/pom.xml
+++ b/pom.xml
@@ -112,7 +112,7 @@ under the License.
<maven.compiler.target>${java.version}</maven.compiler.target>
<scala.macros.version>2.1.0</scala.macros.version>
<!-- Default scala versions, may be overwritten by build profiles -->
- <scala.version>2.11.11</scala.version>
+ <scala.version>2.11.12</scala.version>
<scala.binary.version>2.11</scala.binary.version>
<chill.version>0.7.4</chill.version>
<zookeeper.version>3.4.10</zookeeper.version>
[3/5] flink git commit: [FLINK-5506] [gelly] Fix CommunityDetection
NullPointerException
Posted by gr...@apache.org.
[FLINK-5506] [gelly] Fix CommunityDetection NullPointerException
Double.MIN_VALUE != min(double)
This closes #5126
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/a355df6e
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/a355df6e
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/a355df6e
Branch: refs/heads/master
Commit: a355df6e33f402beac01c2908cb0c64cfeccadb2
Parents: 1a98e32
Author: Greg Hogan <co...@greghogan.com>
Authored: Tue Dec 5 12:38:50 2017 -0500
Committer: Greg Hogan <co...@greghogan.com>
Committed: Thu Dec 21 10:35:53 2017 -0500
----------------------------------------------------------------------
.../flink/graph/library/CommunityDetection.java | 2 +-
.../graph/library/CommunityDetectionTest.java | 116 +++++++++++++++++++
2 files changed, 117 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/a355df6e/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/library/CommunityDetection.java
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/library/CommunityDetection.java b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/library/CommunityDetection.java
index ccf2bb1..2629105 100644
--- a/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/library/CommunityDetection.java
+++ b/flink-libraries/flink-gelly/src/main/java/org/apache/flink/graph/library/CommunityDetection.java
@@ -144,7 +144,7 @@ public class CommunityDetection<K> implements GraphAlgorithm<K, Long, Double, Gr
if (receivedLabelsWithScores.size() > 0) {
// find the label with the highest score from the ones received
- double maxScore = Double.MIN_VALUE;
+ double maxScore = -Double.MAX_VALUE;
long maxScoreLabel = vertex.getValue().f0;
for (long curLabel : receivedLabelsWithScores.keySet()) {
http://git-wip-us.apache.org/repos/asf/flink/blob/a355df6e/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/library/CommunityDetectionTest.java
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/library/CommunityDetectionTest.java b/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/library/CommunityDetectionTest.java
new file mode 100644
index 0000000..cbabcfe
--- /dev/null
+++ b/flink-libraries/flink-gelly/src/test/java/org/apache/flink/graph/library/CommunityDetectionTest.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.flink.graph.library;
+
+import org.apache.flink.api.common.typeinfo.TypeHint;
+import org.apache.flink.graph.Edge;
+import org.apache.flink.graph.Graph;
+import org.apache.flink.graph.Vertex;
+import org.apache.flink.graph.asm.AsmTestBase;
+import org.apache.flink.graph.asm.dataset.ChecksumHashCode;
+import org.apache.flink.graph.asm.dataset.ChecksumHashCode.Checksum;
+import org.apache.flink.graph.generator.SingletonEdgeGraph;
+import org.apache.flink.test.util.TestBaseUtils;
+import org.apache.flink.types.IntValue;
+import org.apache.flink.types.LongValue;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Tests for {@link CommunityDetection}.
+ */
+public class CommunityDetectionTest extends AsmTestBase {
+
+ @Test
+ public void testWithSimpleGraph() throws Exception {
+ Graph<IntValue, Long, Double> result = undirectedSimpleGraph
+ .mapVertices(v -> (long) v.getId().getValue(),
+ new TypeHint<Vertex<IntValue, Long>>(){}.getTypeInfo())
+ .mapEdges(e -> (double) e.getTarget().getValue() + e.getSource().getValue(),
+ new TypeHint<Edge<IntValue, Double>>(){}.getTypeInfo())
+ .run(new CommunityDetection<>(10, 0.5));
+
+ String expectedResult =
+ "(0,3)\n" +
+ "(1,5)\n" +
+ "(2,5)\n" +
+ "(3,3)\n" +
+ "(4,5)\n" +
+ "(5,5)\n";
+
+ TestBaseUtils.compareResultAsText(result.getVertices().collect(), expectedResult);
+ }
+
+ @Test
+ public void testWithSingletonEdgeGraph() throws Exception {
+ Graph<LongValue, Long, Double> result = new SingletonEdgeGraph(env, 1)
+ .generate()
+ .mapVertices(v -> v.getId().getValue(),
+ new TypeHint<Vertex<LongValue, Long>>(){}.getTypeInfo())
+ .mapEdges(e -> 1.0,
+ new TypeHint<Edge<LongValue, Double>>(){}.getTypeInfo())
+ .run(new CommunityDetection<>(10, 0.5));
+
+ String expectedResult =
+ "(0,0)\n" +
+ "(1,1)\n";
+
+ TestBaseUtils.compareResultAsText(result.getVertices().collect(), expectedResult);
+ }
+
+ @Test
+ public void testWithEmptyGraphWithVertices() throws Exception {
+ emptyGraphWithVertices
+ .mapVertices(v -> 0L,
+ new TypeHint<Vertex<LongValue, Long>>(){}.getTypeInfo())
+ .mapEdges(e -> 0.0,
+ new TypeHint<Edge<LongValue, Double>>(){}.getTypeInfo())
+ .run(new CommunityDetection<>(10, 0.5));
+ }
+
+ @Test
+ public void testWithEmptyGraphWithoutVertices() throws Exception {
+ emptyGraphWithoutVertices
+ .mapVertices(v -> 0L,
+ new TypeHint<Vertex<LongValue, Long>>(){}.getTypeInfo())
+ .mapEdges(e -> 0.0,
+ new TypeHint<Edge<LongValue, Double>>(){}.getTypeInfo())
+ .run(new CommunityDetection<>(10, 0.5));
+ }
+
+ @Test
+ public void testWithRMatGraph() throws Exception {
+ Graph<LongValue, Long, Double> result = undirectedRMatGraph(8, 4)
+ .mapVertices(v -> v.getId().getValue(),
+ new TypeHint<Vertex<LongValue, Long>>(){}.getTypeInfo())
+ .mapEdges(e -> (double) e.getTarget().getValue() - e.getSource().getValue(),
+ new TypeHint<Edge<LongValue, Double>>(){}.getTypeInfo())
+ .run(new CommunityDetection<>(10, 0.5));
+
+ Checksum checksum = new ChecksumHashCode<Vertex<LongValue, Long>>()
+ .run(result.getVertices())
+ .execute();
+
+ assertEquals(184, checksum.getCount());
+ assertEquals(0x00000000000cdc96L, checksum.getChecksum());
+ }
+}