You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by fh...@apache.org on 2016/04/15 19:39:46 UTC
[01/12] flink git commit: [FLINK-3700] [core] Remove Guava dependency
from flink-core
Repository: flink
Updated Branches:
refs/heads/master 1b93b3242 -> d938c5f59
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-runtime/src/test/java/org/apache/flink/runtime/util/MathUtilTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/util/MathUtilTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/util/MathUtilTest.java
deleted file mode 100644
index 6ffb64c..0000000
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/util/MathUtilTest.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.util;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.fail;
-
-import org.junit.Test;
-
-public class MathUtilTest {
-
- @Test
- public void testLog2Computation() {
- assertEquals(0, MathUtils.log2floor(1));
- assertEquals(1, MathUtils.log2floor(2));
- assertEquals(1, MathUtils.log2floor(3));
- assertEquals(2, MathUtils.log2floor(4));
- assertEquals(2, MathUtils.log2floor(5));
- assertEquals(2, MathUtils.log2floor(7));
- assertEquals(3, MathUtils.log2floor(8));
- assertEquals(3, MathUtils.log2floor(9));
- assertEquals(4, MathUtils.log2floor(16));
- assertEquals(4, MathUtils.log2floor(17));
- assertEquals(13, MathUtils.log2floor((0x1 << 13) + 1));
- assertEquals(30, MathUtils.log2floor(Integer.MAX_VALUE));
- assertEquals(31, MathUtils.log2floor(-1));
-
- try {
- MathUtils.log2floor(0);
- fail();
- }
- catch (ArithmeticException aex) {}
- }
-
- @Test
- public void testRoundDownToPowerOf2() {
- assertEquals(0, MathUtils.roundDownToPowerOf2(0));
- assertEquals(1, MathUtils.roundDownToPowerOf2(1));
- assertEquals(2, MathUtils.roundDownToPowerOf2(2));
- assertEquals(2, MathUtils.roundDownToPowerOf2(3));
- assertEquals(4, MathUtils.roundDownToPowerOf2(4));
- assertEquals(4, MathUtils.roundDownToPowerOf2(5));
- assertEquals(4, MathUtils.roundDownToPowerOf2(6));
- assertEquals(4, MathUtils.roundDownToPowerOf2(7));
- assertEquals(8, MathUtils.roundDownToPowerOf2(8));
- assertEquals(8, MathUtils.roundDownToPowerOf2(9));
- assertEquals(8, MathUtils.roundDownToPowerOf2(15));
- assertEquals(16, MathUtils.roundDownToPowerOf2(16));
- assertEquals(16, MathUtils.roundDownToPowerOf2(17));
- assertEquals(16, MathUtils.roundDownToPowerOf2(31));
- assertEquals(32, MathUtils.roundDownToPowerOf2(32));
- assertEquals(32, MathUtils.roundDownToPowerOf2(33));
- assertEquals(32, MathUtils.roundDownToPowerOf2(42));
- assertEquals(32, MathUtils.roundDownToPowerOf2(63));
- assertEquals(64, MathUtils.roundDownToPowerOf2(64));
- assertEquals(64, MathUtils.roundDownToPowerOf2(125));
- assertEquals(16384, MathUtils.roundDownToPowerOf2(25654));
- assertEquals(33554432, MathUtils.roundDownToPowerOf2(34366363));
- assertEquals(33554432, MathUtils.roundDownToPowerOf2(63463463));
- assertEquals(1073741824, MathUtils.roundDownToPowerOf2(1852987883));
- assertEquals(1073741824, MathUtils.roundDownToPowerOf2(Integer.MAX_VALUE));
- }
-
- @Test
- public void testPowerOfTwo() {
- assertTrue(MathUtils.isPowerOf2(1));
- assertTrue(MathUtils.isPowerOf2(2));
- assertTrue(MathUtils.isPowerOf2(4));
- assertTrue(MathUtils.isPowerOf2(8));
- assertTrue(MathUtils.isPowerOf2(32768));
- assertTrue(MathUtils.isPowerOf2(65536));
- assertTrue(MathUtils.isPowerOf2(1 << 30));
- assertTrue(MathUtils.isPowerOf2(1L + Integer.MAX_VALUE));
- assertTrue(MathUtils.isPowerOf2(1L << 41));
- assertTrue(MathUtils.isPowerOf2(1L << 62));
-
- assertFalse(MathUtils.isPowerOf2(3));
- assertFalse(MathUtils.isPowerOf2(5));
- assertFalse(MathUtils.isPowerOf2(567923));
- assertFalse(MathUtils.isPowerOf2(Integer.MAX_VALUE));
- assertFalse(MathUtils.isPowerOf2(Long.MAX_VALUE));
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-runtime/src/test/java/org/apache/flink/runtime/util/UnionIteratorTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/util/UnionIteratorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/util/UnionIteratorTest.java
deleted file mode 100644
index 2a79b1f..0000000
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/util/UnionIteratorTest.java
+++ /dev/null
@@ -1,142 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.util;
-
-import org.apache.flink.util.TraversableOnceException;
-import org.junit.Test;
-
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.NoSuchElementException;
-
-import static org.junit.Assert.*;
-
-public class UnionIteratorTest {
-
- @Test
- public void testUnion() {
- try {
- UnionIterator<Integer> iter = new UnionIterator<>();
-
- // should succeed and be empty
- assertFalse(iter.iterator().hasNext());
-
- iter.clear();
-
- try {
- iter.iterator().next();
- fail("should fail with an exception");
- } catch (NoSuchElementException e) {
- // expected
- }
-
- iter.clear();
- iter.addList(Arrays.asList(1, 2, 3, 4, 5, 6, 7));
- iter.addList(Collections.<Integer>emptyList());
- iter.addList(Arrays.asList(8, 9, 10, 11));
-
- int val = 1;
- for (int i : iter) {
- assertEquals(val++, i);
- }
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testTraversableOnce() {
- try {
- UnionIterator<Integer> iter = new UnionIterator<>();
-
- // should succeed
- iter.iterator();
-
- // should fail
- try {
- iter.iterator();
- fail("should fail with an exception");
- } catch (TraversableOnceException e) {
- // expected
- }
-
- // should fail again
- try {
- iter.iterator();
- fail("should fail with an exception");
- } catch (TraversableOnceException e) {
- // expected
- }
-
- // reset the thing, keep it empty
- iter.clear();
-
- // should succeed
- iter.iterator();
-
- // should fail
- try {
- iter.iterator();
- fail("should fail with an exception");
- } catch (TraversableOnceException e) {
- // expected
- }
-
- // should fail again
- try {
- iter.iterator();
- fail("should fail with an exception");
- } catch (TraversableOnceException e) {
- // expected
- }
-
- // reset the thing, add some data
- iter.clear();
- iter.addList(Arrays.asList(1, 2, 3, 4, 5, 6, 7));
-
- // should succeed
- Iterator<Integer> ints = iter.iterator();
- assertNotNull(ints.next());
- assertNotNull(ints.next());
- assertNotNull(ints.next());
-
- // should fail if called in the middle of operations
- try {
- iter.iterator();
- fail("should fail with an exception");
- } catch (TraversableOnceException e) {
- // expected
- }
-
- // reset the thing, keep it empty
- iter.clear();
-
- // should succeed again
- assertFalse(iter.iterator().hasNext());
-
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/SocketTextStreamFunction.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/SocketTextStreamFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/SocketTextStreamFunction.java
index 4211806..21e4394 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/SocketTextStreamFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/SocketTextStreamFunction.java
@@ -18,7 +18,7 @@
package org.apache.flink.streaming.api.functions.source;
import org.apache.flink.annotation.PublicEvolving;
-import org.apache.flink.runtime.util.IOUtils;
+import org.apache.flink.util.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AbstractAlignedProcessingTimeWindowOperator.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AbstractAlignedProcessingTimeWindowOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AbstractAlignedProcessingTimeWindowOperator.java
index af341a6..0a8c6b5 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AbstractAlignedProcessingTimeWindowOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AbstractAlignedProcessingTimeWindowOperator.java
@@ -26,7 +26,7 @@ import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.core.memory.DataInputView;
import org.apache.flink.runtime.state.StateHandle;
-import org.apache.flink.runtime.util.MathUtils;
+import org.apache.flink.util.MathUtils;
import org.apache.flink.streaming.api.operators.AbstractUdfStreamOperator;
import org.apache.flink.streaming.api.operators.TimestampedCollector;
import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingKeyedTimePanes.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingKeyedTimePanes.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingKeyedTimePanes.java
index 9b353fe..79ef4c6 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingKeyedTimePanes.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingKeyedTimePanes.java
@@ -20,7 +20,7 @@ package org.apache.flink.streaming.runtime.operators.windowing;
import org.apache.flink.annotation.Internal;
import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.runtime.util.UnionIterator;
+import org.apache.flink.util.UnionIterator;
import org.apache.flink.streaming.api.functions.windowing.WindowFunction;
import org.apache.flink.streaming.api.operators.AbstractStreamOperator;
import org.apache.flink.streaming.api.windowing.windows.TimeWindow;
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/KeyMap.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/KeyMap.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/KeyMap.java
index 67a2aac..52c321c 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/KeyMap.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/KeyMap.java
@@ -20,7 +20,7 @@ package org.apache.flink.streaming.runtime.operators.windowing;
import org.apache.flink.annotation.Internal;
import org.apache.flink.api.common.functions.ReduceFunction;
-import org.apache.flink.runtime.util.MathUtils;
+import org.apache.flink.util.MathUtils;
import java.util.Arrays;
import java.util.Comparator;
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/partitioner/HashPartitioner.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/partitioner/HashPartitioner.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/partitioner/HashPartitioner.java
index 82f0141..3c93fb7 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/partitioner/HashPartitioner.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/partitioner/HashPartitioner.java
@@ -20,7 +20,7 @@ package org.apache.flink.streaming.runtime.partitioner;
import org.apache.flink.annotation.Internal;
import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.runtime.plugable.SerializationDelegate;
-import org.apache.flink.runtime.util.MathUtils;
+import org.apache.flink.util.MathUtils;
import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
/**
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/IterateTest.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/IterateTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/IterateTest.java
index 27a1e3c..c4343f6 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/IterateTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/IterateTest.java
@@ -29,7 +29,7 @@ import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.runtime.jobgraph.JobGraph;
import org.apache.flink.runtime.jobgraph.JobVertex;
-import org.apache.flink.runtime.util.MathUtils;
+import org.apache.flink.util.MathUtils;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.DataStreamSink;
import org.apache.flink.streaming.api.datastream.IterativeStream;
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/StreamingOperatorsITCase.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/StreamingOperatorsITCase.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/StreamingOperatorsITCase.java
index a77eddd..4f11993 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/StreamingOperatorsITCase.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/StreamingOperatorsITCase.java
@@ -23,7 +23,7 @@ import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.functions.RichMapFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.core.fs.FileSystem;
-import org.apache.flink.runtime.util.MathUtils;
+import org.apache.flink.util.MathUtils;
import org.apache.flink.streaming.api.collector.selector.OutputSelector;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.SplitStream;
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/StreamingOperatorsITCase.scala
----------------------------------------------------------------------
diff --git a/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/StreamingOperatorsITCase.scala b/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/StreamingOperatorsITCase.scala
index 0988e41..d353468 100644
--- a/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/StreamingOperatorsITCase.scala
+++ b/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/StreamingOperatorsITCase.scala
@@ -20,10 +20,10 @@ package org.apache.flink.streaming.api.scala
import org.apache.flink.api.common.functions.{RichMapFunction, FoldFunction}
import org.apache.flink.core.fs.FileSystem
-import org.apache.flink.runtime.util.MathUtils
import org.apache.flink.streaming.api.functions.source.SourceFunction
import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext
import org.apache.flink.test.util.TestBaseUtils
+import org.apache.flink.util.MathUtils
import org.junit.rules.TemporaryFolder
import org.junit.{After, Before, Rule, Test}
[10/12] flink git commit: [FLINK-3738] [table] Refactor
TableEnvironments. Remove Translators and TranslationContext.
Posted by fh...@apache.org.
[FLINK-3738] [table] Refactor TableEnvironments. Remove Translators and TranslationContext.
- Removes Translators (JavaBatchTranslator, ScalaBatchTranslator, JavaStreamTranslator, ScalaStreamTranslator)
- Removes TranslationContext
- Creates TableEnvironments for Java/Scala DataSet/DataStream APIs
- Functionality of translators and translation context is moved to table environments
- Tables are bound to a TableEnvironment
- Simplifies table translation from and to DataSet / DataStream.
- Updates documentation and cleaned up ScalaDocs
This closes #1887
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/1acd8445
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/1acd8445
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/1acd8445
Branch: refs/heads/master
Commit: 1acd844539d3e0ea19d492d2f5f89e32f29827d8
Parents: 760a0d9
Author: Fabian Hueske <fh...@apache.org>
Authored: Tue Apr 12 18:48:38 2016 +0200
Committer: Fabian Hueske <fh...@apache.org>
Committed: Fri Apr 15 19:38:15 2016 +0200
----------------------------------------------------------------------
docs/apis/batch/libs/table.md | 266 ++++++------
.../flink/api/java/table/package-info.java | 23 +-
.../flink/examples/java/JavaTableExample.java | 5 +-
.../api/java/table/BatchTableEnvironment.scala | 165 ++++++++
.../api/java/table/JavaBatchTranslator.scala | 116 ------
.../api/java/table/JavaStreamTranslator.scala | 99 -----
.../api/java/table/StreamTableEnvironment.scala | 123 ++++++
.../flink/api/java/table/TableEnvironment.scala | 160 -------
.../api/scala/table/BatchTableEnvironment.scala | 142 +++++++
.../api/scala/table/DataSetConversions.scala | 78 ++--
.../api/scala/table/DataStreamConversions.scala | 72 ++--
.../api/scala/table/ScalaBatchTranslator.scala | 54 ---
.../api/scala/table/ScalaStreamTranslator.scala | 51 ---
.../scala/table/StreamTableEnvironment.scala | 102 +++++
.../api/scala/table/TableConversions.scala | 50 +--
.../api/scala/table/TableEnvironment.scala | 126 ------
.../apache/flink/api/scala/table/package.scala | 108 +++--
.../api/table/AbstractTableEnvironment.scala | 101 -----
.../flink/api/table/BatchTableEnvironment.scala | 236 +++++++++++
.../api/table/StreamTableEnvironment.scala | 196 +++++++++
.../flink/api/table/TableEnvironment.scala | 358 ++++++++++++++++
.../flink/api/table/plan/PlanTranslator.scala | 80 ----
.../api/table/plan/RexNodeTranslator.scala | 18 +-
.../api/table/plan/TranslationContext.scala | 219 ----------
.../org/apache/flink/api/table/table.scala | 152 +++----
.../flink/examples/scala/TPCHQuery3Table.scala | 8 +-
.../flink/examples/scala/WordCountTable.scala | 4 +-
.../api/java/table/test/AggregationsITCase.java | 20 +-
.../flink/api/java/table/test/AsITCase.java | 416 ------------------
.../api/java/table/test/CastingITCase.java | 16 +-
.../api/java/table/test/DistinctITCase.java | 7 +-
.../api/java/table/test/ExpressionsITCase.java | 9 +-
.../flink/api/java/table/test/FilterITCase.java | 18 +-
.../api/java/table/test/FromDataSetITCase.java | 417 +++++++++++++++++++
.../table/test/GroupedAggregationsITCase.java | 13 +-
.../flink/api/java/table/test/JoinITCase.java | 35 +-
.../api/java/table/test/PojoGroupingITCase.java | 5 +-
.../java/table/test/RegisterDataSetITCase.java | 142 -------
.../flink/api/java/table/test/SelectITCase.java | 13 +-
.../api/java/table/test/SqlExplainITCase.java | 162 -------
.../api/java/table/test/SqlExplainTest.java | 154 +++++++
.../table/test/StringExpressionsITCase.java | 21 +-
.../java/table/test/TableEnvironmentITCase.java | 151 +++++++
.../flink/api/java/table/test/UnionITCase.java | 36 +-
.../api/scala/sql/test/AggregationsITCase.scala | 60 ++-
.../flink/api/scala/sql/test/FilterITCase.scala | 45 +-
.../flink/api/scala/sql/test/JoinITCase.scala | 120 +++---
.../flink/api/scala/sql/test/SelectITCase.scala | 34 +-
.../api/scala/sql/test/TableWithSQLITCase.scala | 22 +-
.../flink/api/scala/sql/test/UnionITCase.scala | 23 +-
.../table/streaming/test/FilterITCase.scala | 22 +-
.../table/streaming/test/SelectITCase.scala | 29 +-
.../table/streaming/test/UnionITCase.scala | 40 +-
.../scala/table/test/AggregationsITCase.scala | 46 +-
.../flink/api/scala/table/test/AsITCase.scala | 151 -------
.../flink/api/scala/table/test/CalcITCase.scala | 31 +-
.../api/scala/table/test/CastingITCase.scala | 30 +-
.../api/scala/table/test/DistinctITCase.scala | 8 +-
.../scala/table/test/ExpressionsITCase.scala | 44 +-
.../api/scala/table/test/FilterITCase.scala | 54 ++-
.../table/test/GroupedAggregationsITCase.scala | 42 +-
.../flink/api/scala/table/test/JoinITCase.scala | 105 +++--
.../table/test/RegisterDataSetITCase.scala | 136 ------
.../api/scala/table/test/SelectITCase.scala | 45 +-
.../api/scala/table/test/SqlExplainTest.scala | 59 ++-
.../table/test/StringExpressionsITCase.scala | 20 +-
.../table/test/TableEnvironmentITCase.scala | 143 +++++++
.../api/scala/table/test/ToTableITCase.scala | 159 +++++++
.../api/scala/table/test/UnionITCase.scala | 74 +++-
.../table/test/utils/ExpressionEvaluator.scala | 31 +-
.../test/utils/TableProgramsTestBase.scala | 34 +-
71 files changed, 3436 insertions(+), 2918 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/docs/apis/batch/libs/table.md
----------------------------------------------------------------------
diff --git a/docs/apis/batch/libs/table.md b/docs/apis/batch/libs/table.md
index 56e2b6b..5b1280c 100644
--- a/docs/apis/batch/libs/table.md
+++ b/docs/apis/batch/libs/table.md
@@ -31,13 +31,11 @@ under the License.
-->
-**The Table API: an experimental feature**
+**Table API and SQL are experimental features**
-Flink's Table API is a SQL-like expression language embedded in Java and Scala.
-Instead of manipulating a `DataSet` or `DataStream`, you can create and work with a relational `Table` abstraction.
-Tables have a schema and allow running relational operations on them, including selection, aggregation, and joins.
-A `Table` can be created from a `DataSet` or a `DataStream` and then queried either using the Table API operators or using SQL queries.
-Once a `Table` is converted back to a `DataSet` or `DataStream`, the defined relational plan is optimized using [Apache Calcite](https://calcite.apache.org/)
+The Table API is a SQL-like expression language that can be embedded in Flink's DataSet and DataStream APIs (Java and Scala).
+A `DataSet` or `DataStream` can be converted into a relational `Table` abstraction. You can apply relational operators such as selection, aggregation, and joins on `Table`s or query them with regular SQL queries.
+When a `Table` is converted back into a `DataSet` or `DataStream`, the logical plan, which was defined by relational operators and SQL queries, is optimized using [Apache Calcite](https://calcite.apache.org/)
and transformed into a `DataSet` or `DataStream` execution plan.
* This will be replaced by the TOC
@@ -46,7 +44,7 @@ and transformed into a `DataSet` or `DataStream` execution plan.
Using the Table API and SQL
----------------------------
-The Table API and SQL are part of the *flink-libraries* Maven project.
+The Table API and SQL are part of the *flink-table* Maven project.
The following dependency must be added to your project in order to use the Table API and SQL:
{% highlight xml %}
@@ -61,58 +59,73 @@ Note that the Table API is currently not part of the binary distribution. See li
Table API
----------
-The Table API provides methods for running relational operations on Tables, both in Scala and Java.
-In the following sections you can find examples that show how to create Tables, how to define and execute relational queries on them,
-and how to retrieve the result of a query as a `DataSet`.
+The Table API provides methods to apply relational operations on DataSets, both in Scala and Java.
+
+The central concept of the Table API is a `Table` which represents a table with relational schema (or relation). Tables can be created from a `DataSet`, converted into a `DataSet`, or registered in a table catalog using a `TableEnvironment`. A `Table` is always bound to a specific `TableEnvironment`. It is not possible to combine Tables of different TableEnvironments.
+
+The following sections show by example how to use the Table API embedded in the Scala and Java DataSet APIs.
### Scala Table API
-The Table API can be enabled by importing `org.apache.flink.api.scala.table._`. This enables
-implicit conversions that allow
-converting a DataSet to a Table. This example shows how a DataSet can
-be converted, how relational queries can be specified and how a Table can be
-converted back to a DataSet:
+The Table API is enabled by importing `org.apache.flink.api.scala.table._`. This enables
+implicit conversions to convert a DataSet to a Table. The following example shows:
+
+- how a `DataSet` is converted to a `Table`,
+- how relational queries are specified, and
+- how a `Table` is converted back to a `DataSet`.
{% highlight scala %}
import org.apache.flink.api.scala._
import org.apache.flink.api.scala.table._
case class WC(word: String, count: Int)
+
+val env = ExecutionEnvironment.getExecutionEnvironment
+val tEnv = TableEnvironment.getTableEnvironment(env)
+
val input = env.fromElements(WC("hello", 1), WC("hello", 1), WC("ciao", 1))
-val expr = input.toTable
-val result = expr.groupBy('word).select('word, 'count.sum as 'count).toDataSet[WC]
+val expr = input.toTable(tEnv)
+val result = expr
+ .groupBy('word)
+ .select('word, 'count.sum as 'count)
+ .toDataSet[WC]
{% endhighlight %}
The expression DSL uses Scala symbols to refer to field names and code generation to
transform expressions to efficient runtime code. Please note that the conversion to and from
-Tables only works when using Scala case classes or Flink POJOs. Please check out
-the [Type Extraction and Serialization]({{ site.baseurl }}/internals/types_serialization.html) section
-to learn the requirements for a class to be considered a POJO.
+Tables only works when using Scala case classes or Java POJOs. Please refer to the [Type Extraction and Serialization]({{ site.baseurl }}/internals/types_serialization.html) section
+to learn the characteristics of a valid POJO.
-This is another example that shows how you
-can join two Tables:
+Another example shows how to join two Tables:
{% highlight scala %}
case class MyResult(a: String, d: Int)
-val input1 = env.fromElements(...).toTable.as('a, 'b)
-val input2 = env.fromElements(...).toTable.as('c, 'd)
-val joined = input1.join(input2).where("a = c && d > 42").select("a, d").toDataSet[MyResult]
+val input1 = env.fromElements(...).toTable(tEnv).as('a, 'b)
+val input2 = env.fromElements(...).toTable(tEnv, 'c, 'd)
+
+val joined = input1.join(input2)
+ .where("a = c && d > 42")
+ .select("a, d")
+ .toDataSet[MyResult]
{% endhighlight %}
-Notice, how a DataSet can be converted to a Table by using `as` and specifying new
-names for the fields. This can also be used to disambiguate fields before a join operation. Also,
-in this example we see that you can also use Strings to specify relational expressions.
+Notice, how the field names of a Table can be changed with `as()` or specified with `toTable()` when converting a DataSet to a Table. In addition, the example shows how to use Strings to specify relational expressions.
-Please refer to the Scaladoc (and Javadoc) for a full list of supported operations and a
-description of the expression syntax.
+Please refer to the Scaladoc (and Javadoc) for a full list of supported operations and a description of the expression syntax.
{% top %}
### Java Table API
-When using Java, Tables can be converted to and from DataSet using `TableEnvironment`.
-This example is equivalent to the above Scala Example:
+When using Flink's Java DataSet API, DataSets are converted to Tables and Tables to DataSets using a `TableEnvironment`.
+The following example shows:
+
+- how a `DataSet` is converted to a `Table`,
+- how relational queries are specified, and
+- how a `Table` is converted back to a `DataSet`.
+
+It is equivalent to the Scala example in the previous section.
{% highlight java %}
@@ -130,15 +143,15 @@ public class WC {
...
-ExecutionEnvironment env = ExecutionEnvironment.createCollectionsEnvironment();
-TableEnvironment tableEnv = new TableEnvironment();
+ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+BatchTableEnvironment tEnv = TableEnvironment.getTableEnvironment(env);
DataSet<WC> input = env.fromElements(
new WC("Hello", 1),
new WC("Ciao", 1),
new WC("Hello", 1));
-Table table = tableEnv.fromDataSet(input);
+Table table = tEnv.fromDataSet(input);
Table wordCounts = table
.groupBy("word")
@@ -147,14 +160,14 @@ Table wordCounts = table
DataSet<WC> result = tableEnv.toDataSet(wordCounts, WC.class);
{% endhighlight %}
-When using Java, the embedded DSL for specifying expressions cannot be used. Only String expressions
-are supported. They support exactly the same feature set as the expression DSL.
+With Java, expressions must be specified by Strings. The embedded expression DSL is not supported.
{% top %}
### Table API Operators
-The Table API provides a domain-spcific language to execute language-integrated queries on structured data in Scala and Java.
-This section gives a brief overview of all available operators. You can find more details of operators in the [Javadoc]({{site.baseurl}}/api/java/org/apache/flink/api/table/Table.html).
+
+The Table API features a domain-specific language to execute language-integrated queries on structured data in Scala and Java.
+This section gives a brief overview of the available operators. You can find more details of operators in the [Javadoc]({{site.baseurl}}/api/java/org/apache/flink/api/table/Table.html).
<div class="codetabs" markdown="1">
<div data-lang="java" markdown="1">
@@ -173,7 +186,7 @@ This section gives a brief overview of all available operators. You can find mor
<tr>
<td><strong>Select</strong></td>
<td>
- <p>Similar to a SQL SELECT statement. Perform a select operation.</p>
+ <p>Similar to a SQL SELECT statement. Performs a select operation.</p>
{% highlight java %}
Table in = tableEnv.fromDataSet(ds, "a, b, c");
Table result = in.select("a, c as d");
@@ -184,7 +197,7 @@ Table result = in.select("a, c as d");
<tr>
<td><strong>As</strong></td>
<td>
- <p>Rename fields.</p>
+ <p>Renames fields.</p>
{% highlight java %}
Table in = tableEnv.fromDataSet(ds, "a, b, c");
Table result = in.as("d, e, f");
@@ -193,51 +206,36 @@ Table result = in.as("d, e, f");
</tr>
<tr>
- <td><strong>Filter</strong></td>
+ <td><strong>Where / Filter</strong></td>
<td>
- <p>Similar to a SQL WHERE clause. Filter out elements that do not pass the filter predicate.</p>
+ <p>Similar to a SQL WHERE clause. Filters out rows that do not pass the filter predicate.</p>
{% highlight java %}
Table in = tableEnv.fromDataSet(ds, "a, b, c");
-Table result = in.filter("a % 2 = 0");
+Table result = in.where("b = 'red'");
{% endhighlight %}
- </td>
- </tr>
-
- <tr>
- <td><strong>Where</strong></td>
- <td>
- <p>Similar to a SQL WHERE clause. Filter out elements that do not pass the filter predicate.</p>
+or
{% highlight java %}
Table in = tableEnv.fromDataSet(ds, "a, b, c");
-Table result = in.where("b = 'red'");
+Table result = in.filter("a % 2 = 0");
{% endhighlight %}
</td>
</tr>
-
<tr>
<td><strong>GroupBy</strong></td>
<td>
- <p>Similar to a SQL GROUPBY clause. Group the elements on the grouping keys, with a following aggregation
- operator to aggregate on per-group basis.</p>
+ <p>Similar to a SQL GROUPBY clause. Groups the rows on the grouping keys, with a following aggregation
+ operator to aggregate rows group-wise.</p>
{% highlight java %}
Table in = tableEnv.fromDataSet(ds, "a, b, c");
Table result = in.groupBy("a").select("a, b.sum as d");
{% endhighlight %}
- <p><i>Note:</i> Flink can refer to nonaggregated columns in the select list that are not named in
- the groupBy clause, it could be used to get better performance by avoiding unnecessary column sorting and
- grouping while nonaggregated column is cogrouped with columns in groupBy clause. For example:</p>
-{% highlight java %}
-Table in = tableEnv.fromDataSet(ds, "a, b, c");
-Table result = in.groupBy("a").select("a, b, c.sum as d");
-{% endhighlight %}
</td>
</tr>
<tr>
<td><strong>Join</strong></td>
<td>
- <p>Similar to a SQL JOIN clause. Join two tables, both tables must have distinct field name, and the where
- clause is mandatory for join condition.</p>
+ <p>Similar to a SQL JOIN clause. Joins two tables. Both tables must have distinct field names and an equality join predicate must be defined using a where or filter operator.</p>
{% highlight java %}
Table left = tableEnv.fromDataSet(ds1, "a, b, c");
Table right = tableEnv.fromDataSet(ds2, "d, e, f");
@@ -249,7 +247,7 @@ Table result = left.join(right).where("a = d").select("a, b, e");
<tr>
<td><strong>Union</strong></td>
<td>
- <p>Similar to a SQL UNION ALL clause. Union two tables, both tables must have identical schema(field names and types).</p>
+ <p>Similar to a SQL UNION ALL clause. Unions two tables. Both tables must have identical schema, i.e., field names and types.</p>
{% highlight java %}
Table left = tableEnv.fromDataSet(ds1, "a, b, c");
Table right = tableEnv.fromDataSet(ds2, "a, b, c");
@@ -288,9 +286,9 @@ Table result = in.distinct();
<tr>
<td><strong>Select</strong></td>
<td>
- <p>Similar to a SQL SELECT statement. Perform a select operation.</p>
+ <p>Similar to a SQL SELECT statement. Performs a select operation.</p>
{% highlight scala %}
-val in = ds.as('a, 'b, 'c);
+val in = ds.toTable(tableEnv, 'a, 'b, 'c);
val result = in.select('a, 'c as 'd);
{% endhighlight %}
</td>
@@ -299,30 +297,24 @@ val result = in.select('a, 'c as 'd);
<tr>
<td><strong>As</strong></td>
<td>
- <p>Rename fields.</p>
+ <p>Renames fields.</p>
{% highlight scala %}
-val in = ds.as('a, 'b, 'c);
+val in = ds.toTable(tableEnv).as('a, 'b, 'c);
{% endhighlight %}
</td>
</tr>
<tr>
- <td><strong>Filter</strong></td>
+ <td><strong>Where / Filter</strong></td>
<td>
- <p>Similar to a SQL WHERE clause. Filter out elements that do not pass the filter predicate.</p>
+ <p>Similar to a SQL WHERE clause. Filters out rows that do not pass the filter predicate.</p>
{% highlight scala %}
-val in = ds.as('a, 'b, 'c);
+val in = ds.toTable(tableEnv, 'a, 'b, 'c);
val result = in.filter('a % 2 === 0)
{% endhighlight %}
- </td>
- </tr>
-
- <tr>
- <td><strong>Where</strong></td>
- <td>
- <p>Similar to a SQL WHERE clause. Filter out elements that do not pass the filter predicate.</p>
+or
{% highlight scala %}
-val in = ds.as('a, 'b, 'c);
+val in = ds.toTable(tableEnv, 'a, 'b, 'c);
val result = in.where('b === "red");
{% endhighlight %}
</td>
@@ -331,30 +323,22 @@ val result = in.where('b === "red");
<tr>
<td><strong>GroupBy</strong></td>
<td>
- <p>Similar to a SQL GROUPBY clause. Group the elements on the grouping keys, with a following aggregation
- operator to aggregate on per-group basis.</p>
+ <p>Similar to a SQL GROUPBY clause. Groups rows on the grouping keys, with a following aggregation
+ operator to aggregate rows group-wise.</p>
{% highlight scala %}
-val in = ds.as('a, 'b, 'c);
+val in = ds.toTable(tableEnv, 'a, 'b, 'c);
val result = in.groupBy('a).select('a, 'b.sum as 'd);
{% endhighlight %}
- <p><i>Note:</i> Flink can refer to nonaggregated columns in the select list that are not named in
- the groupBy clause, it could be used to get better performance by avoiding unnecessary column sorting and
- grouping while nonaggregated column is cogrouped with columns in groupBy clause. For example:</p>
-{% highlight scala %}
-val in = ds.as('a, 'b, 'c);
-val result = in.groupBy('a).select('a, 'b, 'c.sum as 'd);
-{% endhighlight %}
</td>
</tr>
<tr>
<td><strong>Join</strong></td>
<td>
- <p>Similar to a SQL JOIN clause. Join two tables, both tables must have distinct field name, and the where
- clause is mandatory for join condition.</p>
+ <p>Similar to a SQL JOIN clause. Joins two tables. Both tables must have distinct field names and an equality join predicate must be defined using a where or filter operator.</p>
{% highlight scala %}
-val left = ds1.as('a, 'b, 'c);
-val right = ds2.as('d, 'e, 'f);
+val left = ds1.toTable(tableEnv, 'a, 'b, 'c);
+val right = ds2.toTable(tableEnv, 'd, 'e, 'f);
val result = left.join(right).where('a === 'd).select('a, 'b, 'e);
{% endhighlight %}
</td>
@@ -363,10 +347,10 @@ val result = left.join(right).where('a === 'd).select('a, 'b, 'e);
<tr>
<td><strong>Union</strong></td>
<td>
- <p>Similar to a SQL UNION ALL clause. Union two tables, both tables must have identical schema(field names and types).</p>
+ <p>Similar to a SQL UNION ALL clause. Unions two tables, both tables must have identical schema(field names and types).</p>
{% highlight scala %}
-val left = ds1.as('a, 'b, 'c);
-val right = ds2.as('a, 'b, 'c);
+val left = ds1.toTable(tableEnv, 'a, 'b, 'c);
+val right = ds2.toTable(tableEnv, 'a, 'b, 'c);
val result = left.unionAll(right);
{% endhighlight %}
</td>
@@ -377,7 +361,7 @@ val result = left.unionAll(right);
<td>
<p>Similar to a SQL DISTINCT clause. Returns rows with distinct value combinations.</p>
{% highlight scala %}
-val in = ds.as('a, 'b, 'c);
+val in = ds.toTable(tableEnv, 'a, 'b, 'c);
val result = in.distinct();
{% endhighlight %}
</td>
@@ -391,9 +375,7 @@ val result = in.distinct();
{% top %}
### Expression Syntax
-Some of operators in previous section expect an expression. These can either be specified using an embedded Scala DSL or
-a String expression. Please refer to the examples above to learn how expressions can be
-formulated.
+Some of the operators in previous sections expect one or more expressions. Expressions can be specified using an embedded Scala DSL or as Strings. Please refer to the examples above to learn how expressions can be specified.
This is the complete EBNF grammar for expressions:
@@ -438,19 +420,17 @@ SQL
----
The Table API also supports embedded SQL queries.
In order to use a `Table` or `DataSet` in a SQL query, it has to be registered in the `TableEnvironment`, using a unique name.
-A registered `Table` can be retrieved back from the `TableEnvironment` using the `scan` method:
+A registered `Table` can be retrieved back from the `TableEnvironment` using the `scan()` method:
<div class="codetabs" markdown="1">
<div data-lang="java" markdown="1">
{% highlight java %}
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
// create a Table environment
-TableEnvironment tableEnv = new TableEnvironment();
-// reset the translation context: this will erase existing registered Tables
-TranslationContext.reset();
-// read a DataSet from an external source
+BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
+
DataSet<Tuple2<Integer, Long>> ds = env.readCsvFile(...);
-// register the DataSet under the name "MyTable"
+// register the DataSet as table "MyTable"
tableEnv.registerDataSet("MyTable", ds);
// retrieve "MyTable" into a new Table
Table t = tableEnv.scan("MyTable");
@@ -461,12 +441,10 @@ Table t = tableEnv.scan("MyTable");
{% highlight scala %}
val env = ExecutionEnvironment.getExecutionEnvironment
// create a Table environment
-val tableEnv = new TableEnvironment
-// reset the translation context: this will erase existing registered Tables
-TranslationContext.reset()
-// read a DataSet from an external source
+val tableEnv = TableEnvironment.getTableEnvironment(env)
+
val ds = env.readCsvFile(...)
-// register the DataSet under the name "MyTable"
+// register the DataSet as table "MyTable"
tableEnv.registerDataSet("MyTable", ds)
// retrieve "MyTable" into a new Table
val t = tableEnv.scan("MyTable")
@@ -476,20 +454,20 @@ val t = tableEnv.scan("MyTable")
*Note: Table names are not allowed to follow the `^_DataSetTable_[0-9]+` pattern, as this is reserved for internal use only.*
-When registering a `DataSet`, one can also give names to the `Table` columns. For example, if "MyTable" has three columns, `user`, `product`, and `order`, we can give them names upon registering the `DataSet` as shown below:
+When registering a `DataSet`, one can also specify the field names of the table:
<div class="codetabs" markdown="1">
<div data-lang="java" markdown="1">
{% highlight java %}
-// register the DataSet under the name "MyTable" with columns user, product, and order
-tableEnv.registerDataSet("MyTable", ds, "user, product, order");
+// register the DataSet as table "Orders" with fields user, product, and amount
+tableEnv.registerDataSet("Orders", ds, "user, product, amount");
{% endhighlight %}
</div>
<div data-lang="scala" markdown="1">
{% highlight scala %}
-// register the DataSet under the name "MyTable" with columns user, product, and order
-tableEnv.registerDataSet("MyTable", ds, 'user, 'product, 'order)
+// register the DataSet as table "Orders" with fields user, product, and amount
+tableEnv.registerDataSet("Orders", ds, 'user, 'product, 'amount)
{% endhighlight %}
</div>
</div>
@@ -499,61 +477,49 @@ A `Table` can be registered in a similar way:
<div class="codetabs" markdown="1">
<div data-lang="java" markdown="1">
{% highlight java %}
-// read a DataSet from an external source
-DataSet<Tuple2<Integer, Long>> ds = env.readCsvFile(...);
-// create a Table from the DataSet with columns user, product, and order
-Table t = tableEnv.fromDataSet(ds).as("user, product, order");
-// register the Table under the name "MyTable"
-tableEnv.registerTable("MyTable", t);
+Table t = tableEnv.fromDataSet(ds).as("user, product, amount");
+// register the Table as table "Orders"
+tableEnv.registerTable("Orders", t);
{% endhighlight %}
</div>
<div data-lang="scala" markdown="1">
{% highlight scala %}
-// read a DataSet from an external source and
-// create a Table from the DataSet with columns user, product, and order
-val t = env.readCsvFile(...).as('user, 'product, 'order)
-// register the Table under the name "MyTable"
-tableEnv.registerTable("MyTable", t)
+val t = ds.toTable(tableEnv, 'user, 'product, 'amount)
+// register the Table as table "Orders"
+tableEnv.registerTable("Orders", t)
{% endhighlight %}
</div>
</div>
-After registering a `Table` or `DataSet`, one can use them in SQL queries. A SQL query is defined using the `sql` method of the `TableEnvironment`.
-The result of the method is a new `Table` which can either be converted back to a `DataSet` or used in subsequent Table API queries.
+Registered tables can be used in SQL queries. A SQL query is defined using the `sql()` method of the `TableEnvironment`. It returns a new `Table` which can be converted back to a `DataSet` or used in subsequent Table API queries.
<div class="codetabs" markdown="1">
<div data-lang="java" markdown="1">
{% highlight java %}
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-// create a Table environment
-TableEnvironment tableEnv = new TableEnvironment();
-// reset the translation context: this will erase existing registered Tables
-TranslationContext.reset();
+BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
+
// read a DataSet from an external source
DataSet<Tuple2<Integer, Long>> ds = env.readCsvFile(...);
-// create a Table from the DataSet
-Table t = tableEnv.fromDataSet(ds);
-// register the Table under the name "MyTable"
-tableEnv.registerTable("MyTable", t);
-// run a sql query and retrieve the result in a new Table
-Table result = tableEnv.sql("SELECT * FROM MyTable");
+// register the DataSet as table "Orders"
+tableEnv.registerDataSet("Orders", ds, "user, product, amount");
+// run a SQL query and retrieve the result in a new Table
+Table result = tableEnv.sql("SELECT SUM(amount) FROM Orders WHERE product = 10");
{% endhighlight %}
</div>
<div data-lang="scala" markdown="1">
{% highlight scala %}
val env = ExecutionEnvironment.getExecutionEnvironment
-// create a Table environment
-val tableEnv = new TableEnvironment
-// reset the translation context: this will erase existing registered Tables
-TranslationContext.reset()
-// create a Table
-val t = env.readCsvFile(...).as('a, 'b, 'c)
-// register the Table under the name "MyTable"
-tableEnv.registerTable("MyTable", t)
-// run a sql query and retrieve the result in a new Table
-val result = tableEnv.sql("SELECT * FROM MyTable")
+val tableEnv = TableEnvironment.getTableEnvironment(env)
+
+// read a DataSet from an external source
+val ds = env.readCsvFile(...)
+// register the DataSet under the name "Orders"
+tableEnv.registerDataSet("Orders", ds, 'user, 'product, 'amount)
+// run a SQL query and retrieve the result in a new Table
+val result = tableEnv.sql("SELECT SUM(amount) FROM Orders WHERE product = 10")
{% endhighlight %}
</div>
</div>
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/main/java/org/apache/flink/api/java/table/package-info.java
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/java/org/apache/flink/api/java/table/package-info.java b/flink-libraries/flink-table/src/main/java/org/apache/flink/api/java/table/package-info.java
index 97113bb..1db4cb8 100644
--- a/flink-libraries/flink-table/src/main/java/org/apache/flink/api/java/table/package-info.java
+++ b/flink-libraries/flink-table/src/main/java/org/apache/flink/api/java/table/package-info.java
@@ -19,12 +19,14 @@
/**
* <strong>Table API (Java)</strong><br>
*
- * {@link org.apache.flink.api.java.table.TableEnvironment} can be used to create a
- * {@link org.apache.flink.api.table.Table} from a {@link org.apache.flink.api.java.DataSet}
- * or {@link org.apache.flink.streaming.api.datastream.DataStream}.
+ * A {@link org.apache.flink.api.java.table.BatchTableEnvironment} can be used to create a
+ * {@link org.apache.flink.api.table.Table} from a {@link org.apache.flink.api.java.DataSet}.
+ * Equivalently, a {@link org.apache.flink.api.java.table.StreamTableEnvironment} can be used to
+ * create a {@link org.apache.flink.api.table.Table} from a
+ * {@link org.apache.flink.streaming.api.datastream.DataStream}.
*
* <p>
- * This can be used to perform SQL-like queries on data. Please have
+ * Tables can be used to perform SQL-like queries on data. Please have
* a look at {@link org.apache.flink.api.table.Table} to see which operations are supported and
* how query strings are written.
*
@@ -32,29 +34,30 @@
* Example:
*
* <pre>{@code
- * ExecutionEnvironment env = ExecutionEnvironment.createCollectionsEnvironment();
+ * ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ * BatchTableEnvironment tEnv = TableEnvironment.getTableEnvironment(env);
*
* DataSet<WC> input = env.fromElements(
* new WC("Hello", 1),
* new WC("Ciao", 1),
* new WC("Hello", 1));
*
- * Table table = TableUtil.from(input);
+ * Table table = tEnv.fromDataSet(input);
*
* Table filtered = table
* .groupBy("word")
* .select("word.count as count, word")
* .filter("count = 2");
*
- * DataSet<WC> result = TableUtil.toSet(filtered, WC.class);
+ * DataSet<WC> result = tEnv.toDataSet(filtered, WC.class);
*
* result.print();
- * env.execute();
* }</pre>
*
* <p>
* As seen above, a {@link org.apache.flink.api.table.Table} can be converted back to the
- * underlying API representation using {@link org.apache.flink.api.java.table.TableEnvironment#toDataSet(org.apache.flink.api.table.Table, java.lang.Class)}
- * or {@link org.apache.flink.api.java.table.TableEnvironment#toDataStream(org.apache.flink.api.table.Table, java.lang.Class)}}.
+ * underlying API representation using
+ * {@link org.apache.flink.api.java.table.BatchTableEnvironment#toDataSet(org.apache.flink.api.table.Table, java.lang.Class)}
+ * or {@link org.apache.flink.api.java.table.StreamTableEnvironment#toDataStream(org.apache.flink.api.table.Table, java.lang.Class)}}.
*/
package org.apache.flink.api.java.table;
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/main/java/org/apache/flink/examples/java/JavaTableExample.java
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/java/org/apache/flink/examples/java/JavaTableExample.java b/flink-libraries/flink-table/src/main/java/org/apache/flink/examples/java/JavaTableExample.java
index c7e69c9..0a776a4 100644
--- a/flink-libraries/flink-table/src/main/java/org/apache/flink/examples/java/JavaTableExample.java
+++ b/flink-libraries/flink-table/src/main/java/org/apache/flink/examples/java/JavaTableExample.java
@@ -20,7 +20,8 @@ package org.apache.flink.examples.java;
import org.apache.flink.api.table.Table;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.table.TableEnvironment;
+import org.apache.flink.api.java.table.BatchTableEnvironment;
+import org.apache.flink.api.table.TableEnvironment;
/**
* Very simple example that shows how the Java Table API can be used.
@@ -49,7 +50,7 @@ public class JavaTableExample {
public static void main(String[] args) throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.createCollectionsEnvironment();
- TableEnvironment tableEnv = new TableEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
DataSet<WC> input = env.fromElements(
new WC("Hello", 1),
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/java/table/BatchTableEnvironment.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/java/table/BatchTableEnvironment.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/java/table/BatchTableEnvironment.scala
new file mode 100644
index 0000000..69bff95
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/java/table/BatchTableEnvironment.scala
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.api.java.table
+
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.api.java.{ExecutionEnvironment, DataSet}
+import org.apache.flink.api.java.typeutils.TypeExtractor
+import org.apache.flink.api.table.expressions.ExpressionParser
+import org.apache.flink.api.table.{TableConfig, Table}
+
+/**
+ * The [[org.apache.flink.api.table.TableEnvironment]] for a Java batch [[DataSet]]
+ * [[ExecutionEnvironment]].
+ *
+ * A TableEnvironment can be used to:
+ * - convert a [[DataSet]] to a [[Table]]
+ * - register a [[DataSet]] in the [[org.apache.flink.api.table.TableEnvironment]]'s catalog
+ * - register a [[Table]] in the [[org.apache.flink.api.table.TableEnvironment]]'s catalog
+ * - scan a registered table to obtain a [[Table]]
+ * - specify a SQL query on registered tables to obtain a [[Table]]
+ * - convert a [[Table]] into a [[DataSet]]
+ * - explain the AST and execution plan of a [[Table]]
+ *
+ * @param execEnv The Java batch [[ExecutionEnvironment]] of the TableEnvironment.
+ * @param config The configuration of the TableEnvironment.
+ */
+class BatchTableEnvironment(
+ protected val execEnv: ExecutionEnvironment,
+ config: TableConfig)
+ extends org.apache.flink.api.table.BatchTableEnvironment(config) {
+
+ /**
+ * Converts the given [[DataSet]] into a [[Table]].
+ *
+ * The field names of the [[Table]] are automatically derived from the type of the [[DataSet]].
+ *
+ * @param dataSet The [[DataSet]] to be converted.
+ * @tparam T The type of the [[DataSet]].
+ * @return The converted [[Table]].
+ */
+ def fromDataSet[T](dataSet: DataSet[T]): Table = {
+
+ val name = createUniqueTableName()
+ registerDataSetInternal(name, dataSet)
+ scan(name)
+ }
+
+ /**
+ * Converts the given [[DataSet]] into a [[Table]] with specified field names.
+ *
+ * Example:
+ *
+ * {{{
+ * DataSet<Tuple2<String, Long>> set = ...
+ * Table tab = tableEnv.fromDataSet(set, "a, b")
+ * }}}
+ *
+ * @param dataSet The [[DataSet]] to be converted.
+ * @param fields The field names of the resulting [[Table]].
+ * @tparam T The type of the [[DataSet]].
+ * @return The converted [[Table]].
+ */
+ def fromDataSet[T](dataSet: DataSet[T], fields: String): Table = {
+ val exprs = ExpressionParser
+ .parseExpressionList(fields)
+ .toArray
+
+ val name = createUniqueTableName()
+ registerDataSetInternal(name, dataSet, exprs)
+ scan(name)
+ }
+
+ /**
+ * Registers the given [[DataSet]] as table in the
+ * [[org.apache.flink.api.table.TableEnvironment]]'s catalog.
+ * Registered tables can be referenced in SQL queries.
+ *
+ * The field names of the [[Table]] are automatically derived from the type of the [[DataSet]].
+ *
+ * @param name The name under which the [[DataSet]] is registered in the catalog.
+ * @param dataSet The [[DataSet]] to register.
+ * @tparam T The type of the [[DataSet]] to register.
+ */
+ def registerDataSet[T](name: String, dataSet: DataSet[T]): Unit = {
+
+ checkValidTableName(name)
+ registerDataSetInternal(name, dataSet)
+ }
+
+ /**
+ * Registers the given [[DataSet]] as table with specified field names in the
+ * [[org.apache.flink.api.table.TableEnvironment]]'s catalog.
+ * Registered tables can be referenced in SQL queries.
+ *
+ * Example:
+ *
+ * {{{
+ * DataSet<Tuple2<String, Long>> set = ...
+ * tableEnv.registerDataSet("myTable", set, "a, b")
+ * }}}
+ *
+ * @param name The name under which the [[DataSet]] is registered in the catalog.
+ * @param dataSet The [[DataSet]] to register.
+ * @param fields The field names of the registered table.
+ * @tparam T The type of the [[DataSet]] to register.
+ */
+ def registerDataSet[T](name: String, dataSet: DataSet[T], fields: String): Unit = {
+ val exprs = ExpressionParser
+ .parseExpressionList(fields)
+ .toArray
+
+ checkValidTableName(name)
+ registerDataSetInternal(name, dataSet, exprs)
+ }
+
+ /**
+ * Converts the given [[Table]] into a [[DataSet]] of a specified type.
+ *
+ * The fields of the [[Table]] are mapped to [[DataSet]] fields as follows:
+ * - [[org.apache.flink.api.table.Row]] and [[org.apache.flink.api.java.tuple.Tuple]]
+ * types: Fields are mapped by position, field types must match.
+ * - POJO [[DataSet]] types: Fields are mapped by field name, field types must match.
+ *
+ * @param table The [[Table]] to convert.
+ * @param clazz The class of the type of the resulting [[DataSet]].
+ * @tparam T The type of the resulting [[DataSet]].
+ * @return The converted [[DataSet]].
+ */
+ def toDataSet[T](table: Table, clazz: Class[T]): DataSet[T] = {
+ translate[T](table)(TypeExtractor.createTypeInfo(clazz))
+ }
+
+ /**
+ * Converts the given [[Table]] into a [[DataSet]] of a specified type.
+ *
+ * The fields of the [[Table]] are mapped to [[DataSet]] fields as follows:
+ * - [[org.apache.flink.api.table.Row]] and [[org.apache.flink.api.java.tuple.Tuple]]
+ * types: Fields are mapped by position, field types must match.
+ * - POJO [[DataSet]] types: Fields are mapped by field name, field types must match.
+ *
+ * @param table The [[Table]] to convert.
+ * @param typeInfo The [[TypeInformation]] that specifies the type of the resulting [[DataSet]].
+ * @tparam T The type of the resulting [[DataSet]].
+ * @return The converted [[DataSet]].
+ */
+ def toDataSet[T](table: Table, typeInfo: TypeInformation[T]): DataSet[T] = {
+ translate[T](table)(typeInfo)
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/java/table/JavaBatchTranslator.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/java/table/JavaBatchTranslator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/java/table/JavaBatchTranslator.scala
deleted file mode 100644
index 4688c82..0000000
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/java/table/JavaBatchTranslator.scala
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.api.java.table
-
-import org.apache.calcite.plan.RelOptPlanner.CannotPlanException
-import org.apache.calcite.plan.RelOptUtil
-import org.apache.calcite.rel.RelNode
-import org.apache.calcite.sql2rel.RelDecorrelator
-import org.apache.calcite.tools.Programs
-import org.apache.flink.api.common.typeinfo.TypeInformation
-import org.apache.flink.api.java.{DataSet => JavaDataSet}
-import org.apache.flink.api.table.plan._
-import org.apache.flink.api.table.{FlinkPlannerImpl, TableConfig, Table}
-import org.apache.flink.api.table.plan.nodes.dataset.{DataSetConvention, DataSetRel}
-import org.apache.flink.api.table.plan.rules.FlinkRuleSets
-import org.apache.flink.api.table.plan.schema.DataSetTable
-
-/**
- * [[PlanTranslator]] for creating [[Table]]s from Java [[org.apache.flink.api.java.DataSet]]s and
- * translating them back to Java [[org.apache.flink.api.java.DataSet]]s.
- */
-class JavaBatchTranslator(config: TableConfig) extends PlanTranslator {
-
- type Representation[A] = JavaDataSet[A]
-
- override def createTable[A](
- repr: Representation[A],
- fieldIndexes: Array[Int],
- fieldNames: Array[String]): Table = {
-
- // create table representation from DataSet
- val dataSetTable = new DataSetTable[A](
- repr.asInstanceOf[JavaDataSet[A]],
- fieldIndexes,
- fieldNames
- )
-
- val tabName = TranslationContext.registerDataSetTable(dataSetTable)
- val relBuilder = TranslationContext.getRelBuilder
-
- // create table scan operator
- relBuilder.scan(tabName)
- val relNode = relBuilder.build()
- new Table(relNode, relBuilder)
- }
-
- override def translate[A](lPlan: RelNode)(implicit tpe: TypeInformation[A]): JavaDataSet[A] = {
-
- // get the planner for the plan
- val planner = lPlan.getCluster.getPlanner
-
- // decorrelate
- val decorPlan = RelDecorrelator.decorrelateQuery(lPlan)
-
- // optimize the logical Flink plan
- val optProgram = Programs.ofRules(FlinkRuleSets.DATASET_OPT_RULES)
- val flinkOutputProps = lPlan.getTraitSet.replace(DataSetConvention.INSTANCE).simplify()
-
- val dataSetPlan = try {
- optProgram.run(planner, decorPlan, flinkOutputProps)
- }
- catch {
- case e: CannotPlanException =>
- throw new PlanGenException(
- s"Cannot generate a valid execution plan for the given query: \n\n" +
- s"${RelOptUtil.toString(lPlan)}\n" +
- "Please consider filing a bug report.", e)
- case a: AssertionError =>
- throw a.getCause
- }
-
- dataSetPlan match {
- case node: DataSetRel =>
- node.translateToPlan(
- config,
- Some(tpe.asInstanceOf[TypeInformation[Any]])
- ).asInstanceOf[JavaDataSet[A]]
- case _ => ???
- }
-
- }
-
- /**
- * Parse, validate, and translate a SQL query into a relNode Table
- */
- def translateSQL(query: String): Table = {
-
- val frameworkConfig = TranslationContext.getFrameworkConfig
- val planner = new FlinkPlannerImpl(frameworkConfig, TranslationContext.getPlanner)
- // parse the sql query
- val parsed = planner.parse(query)
- // validate the sql query
- val validated = planner.validate(parsed)
- // transform to a relational tree
- val relational = planner.rel(validated)
-
- new Table(relational.rel, TranslationContext.getRelBuilder)
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/java/table/JavaStreamTranslator.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/java/table/JavaStreamTranslator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/java/table/JavaStreamTranslator.scala
deleted file mode 100644
index e4b8ca0..0000000
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/java/table/JavaStreamTranslator.scala
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.api.java.table
-
-import org.apache.calcite.plan.RelOptPlanner.CannotPlanException
-import org.apache.calcite.plan.{RelTraitSet, RelOptUtil}
-import org.apache.calcite.rel.{RelCollations, RelNode}
-import org.apache.calcite.sql2rel.RelDecorrelator
-import org.apache.calcite.tools.Programs
-import org.apache.flink.api.common.typeinfo.TypeInformation
-import org.apache.flink.streaming.api.datastream.{DataStream => JavaDataStream}
-import org.apache.flink.api.table.plan._
-import org.apache.flink.api.table.{TableConfig, Table}
-import org.apache.flink.api.table.plan.nodes.datastream.{DataStreamRel, DataStreamConvention}
-import org.apache.flink.api.table.plan.rules.FlinkRuleSets
-import org.apache.flink.api.table.plan.schema.DataStreamTable
-
-/**
- * [[PlanTranslator]] for creating [[Table]]s
- * from Java [[org.apache.flink.streaming.api.datastream.DataStream]]s and
- * translating them back to Java [[org.apache.flink.streaming.api.datastream.DataStream]]s.
- */
-class JavaStreamTranslator(config: TableConfig) extends PlanTranslator {
-
- type Representation[A] = JavaDataStream[A]
-
- override def createTable[A](
- repr: Representation[A],
- fieldIndexes: Array[Int],
- fieldNames: Array[String]): Table = {
-
- // create table representation from DataSet
- val dataStreamTable = new DataStreamTable[A](
- repr.asInstanceOf[JavaDataStream[A]],
- fieldIndexes,
- fieldNames
- )
-
- val tabName = TranslationContext.addDataStream(dataStreamTable)
- val relBuilder = TranslationContext.getRelBuilder
-
- // create table scan operator
- relBuilder.scan(tabName)
- new Table(relBuilder.build(), relBuilder)
- }
-
- override def translate[A](lPlan: RelNode)(implicit tpe: TypeInformation[A]): JavaDataStream[A] = {
-
- // get the planner for the plan
- val planner = lPlan.getCluster.getPlanner
-
- // decorrelate
- val decorPlan = RelDecorrelator.decorrelateQuery(lPlan)
-
- // optimize the logical Flink plan
- val optProgram = Programs.ofRules(FlinkRuleSets.DATASTREAM_OPT_RULES)
- val flinkOutputProps = RelTraitSet.createEmpty()
- .plus(DataStreamConvention.INSTANCE)
- .plus(RelCollations.of()).simplify()
-
- val dataStreamPlan = try {
- optProgram.run(planner, decorPlan, flinkOutputProps)
- }
- catch {
- case e: CannotPlanException =>
- throw new PlanGenException(
- s"Cannot generate a valid execution plan for the given query: \n\n" +
- s"${RelOptUtil.toString(lPlan)}\n" +
- "Please consider filing a bug report.", e)
- }
-
- dataStreamPlan match {
- case node: DataStreamRel =>
- node.translateToPlan(
- config,
- Some(tpe.asInstanceOf[TypeInformation[Any]])
- ).asInstanceOf[JavaDataStream[A]]
- case _ => ???
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/java/table/StreamTableEnvironment.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/java/table/StreamTableEnvironment.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/java/table/StreamTableEnvironment.scala
new file mode 100644
index 0000000..7479426
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/java/table/StreamTableEnvironment.scala
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.api.java.table
+
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.api.java.typeutils.TypeExtractor
+import org.apache.flink.api.table.{TableConfig, Table}
+import org.apache.flink.api.table.expressions.ExpressionParser
+import org.apache.flink.streaming.api.datastream.DataStream
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment
+
+/**
+ * The [[org.apache.flink.api.table.TableEnvironment]] for a Java [[StreamExecutionEnvironment]].
+ *
+ * A TableEnvironment can be used to:
+ * - convert a [[DataStream]] to a [[Table]]
+ * - register a [[DataStream]] in the [[org.apache.flink.api.table.TableEnvironment]]'s catalog
+ * - register a [[Table]] in the [[org.apache.flink.api.table.TableEnvironment]]'s catalog
+ * - scan a registered table to obtain a [[Table]]
+ * - specify a SQL query on registered tables to obtain a [[Table]]
+ * - convert a [[Table]] into a [[DataStream]]
+ * - explain the AST and execution plan of a [[Table]]
+ *
+ * @param execEnv The Java [[StreamExecutionEnvironment]] of the TableEnvironment.
+ * @param config The configuration of the TableEnvironment.
+ */
+class StreamTableEnvironment(
+ protected val execEnv: StreamExecutionEnvironment,
+ config: TableConfig)
+ extends org.apache.flink.api.table.StreamTableEnvironment(config) {
+
+ /**
+ * Converts the given [[DataStream]] into a [[Table]].
+ *
+ * The field names of the [[Table]] are automatically derived from the type of the
+ * [[DataStream]].
+ *
+ * @param dataStream The [[DataStream]] to be converted.
+ * @tparam T The type of the [[DataStream]].
+ * @return The converted [[Table]].
+ */
+ def fromDataStream[T](dataStream: DataStream[T]): Table = {
+
+ val name = createUniqueTableName()
+ registerDataStreamInternal(name, dataStream)
+ ingest(name)
+ }
+
+ /**
+ * Converts the given [[DataStream]] into a [[Table]] with specified field names.
+ *
+ * Example:
+ *
+ * {{{
+ * DataStream<Tuple2<String, Long>> stream = ...
+ * Table tab = tableEnv.fromDataStream(stream, "a, b")
+ * }}}
+ *
+ * @param dataStream The [[DataStream]] to be converted.
+ * @param fields The field names of the resulting [[Table]].
+ * @tparam T The type of the [[DataStream]].
+ * @return The converted [[Table]].
+ */
+ def fromDataStream[T](dataStream: DataStream[T], fields: String): Table = {
+ val exprs = ExpressionParser
+ .parseExpressionList(fields)
+ .toArray
+
+ val name = createUniqueTableName()
+ registerDataStreamInternal(name, dataStream, exprs)
+ ingest(name)
+ }
+
+ /**
+ * Converts the given [[Table]] into a [[DataStream]] of a specified type.
+ *
+ * The fields of the [[Table]] are mapped to [[DataStream]] fields as follows:
+ * - [[org.apache.flink.api.table.Row]] and [[org.apache.flink.api.java.tuple.Tuple]]
+ * types: Fields are mapped by position, field types must match.
+ * - POJO [[DataStream]] types: Fields are mapped by field name, field types must match.
+ *
+ * @param table The [[Table]] to convert.
+ * @param clazz The class of the type of the resulting [[DataStream]].
+ * @tparam T The type of the resulting [[DataStream]].
+ * @return The converted [[DataStream]].
+ */
+ def toDataStream[T](table: Table, clazz: Class[T]): DataStream[T] = {
+ translate[T](table)(TypeExtractor.createTypeInfo(clazz))
+ }
+
+ /**
+ * Converts the given [[Table]] into a [[DataStream]] of a specified type.
+ *
+ * The fields of the [[Table]] are mapped to [[DataStream]] fields as follows:
+ * - [[org.apache.flink.api.table.Row]] and [[org.apache.flink.api.java.tuple.Tuple]]
+ * types: Fields are mapped by position, field types must match.
+ * - POJO [[DataStream]] types: Fields are mapped by field name, field types must match.
+ *
+ * @param table The [[Table]] to convert.
+ * @param typeInfo The [[TypeInformation]] that specifies the type of the [[DataStream]].
+ * @tparam T The type of the resulting [[DataStream]].
+ * @return The converted [[DataStream]].
+ */
+ def toDataStream[T](table: Table, typeInfo: TypeInformation[T]): DataStream[T] = {
+ translate[T](table)(typeInfo)
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/java/table/TableEnvironment.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/java/table/TableEnvironment.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/java/table/TableEnvironment.scala
deleted file mode 100644
index 90954c6..0000000
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/java/table/TableEnvironment.scala
+++ /dev/null
@@ -1,160 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.api.java.table
-
-import org.apache.flink.api.common.typeinfo.TypeInformation
-import org.apache.flink.api.java.DataSet
-import org.apache.flink.api.java.typeutils.TypeExtractor
-import org.apache.flink.api.table.expressions.ExpressionParser
-import org.apache.flink.api.table.{AbstractTableEnvironment, Table}
-import org.apache.flink.streaming.api.datastream.DataStream
-
-/**
- * Environment for working with the Table API.
- *
- * This can be used to convert a [[DataSet]] or a [[DataStream]] to a [[Table]] and back again. You
- * can also use the provided methods to create a [[Table]] directly from a data source.
- */
-class TableEnvironment extends AbstractTableEnvironment {
-
- /**
- * Transforms the given DataSet to a [[org.apache.flink.api.table.Table]].
- * The fields of the DataSet type are renamed to the given set of fields:
- *
- * Example:
- *
- * {{{
- * tableEnv.fromDataSet(set, "a, b")
- * }}}
- *
- * This will transform the set containing elements of two fields to a table where the fields
- * are named a and b.
- */
- def fromDataSet[T](set: DataSet[T], fields: String): Table = {
- new JavaBatchTranslator(config).createTable(set, fields)
- }
-
- /**
- * Transforms the given DataSet to a [[org.apache.flink.api.table.Table]].
- * The fields of the DataSet type are used to name the
- * [[org.apache.flink.api.table.Table]] fields.
- */
- def fromDataSet[T](set: DataSet[T]): Table = {
- new JavaBatchTranslator(config).createTable(set)
- }
-
- /**
- * Converts the given [[org.apache.flink.api.table.Table]] to
- * a DataSet. The given type must have exactly the same field types and field order as the
- * [[org.apache.flink.api.table.Table]]. Row and tuple types can be mapped by position.
- * POJO types require name equivalence to be mapped correctly as their fields do not have
- * an order.
- */
- @SuppressWarnings(Array("unchecked"))
- def toDataSet[T](table: Table, clazz: Class[T]): DataSet[T] = {
- new JavaBatchTranslator(config).translate[T](table.relNode)(
- TypeExtractor.createTypeInfo(clazz).asInstanceOf[TypeInformation[T]])
- }
-
- /**
- * Converts the given [[org.apache.flink.api.table.Table]] to
- * a DataSet. The given type must have exactly the same field types and field order as the
- * [[org.apache.flink.api.table.Table]]. Row and tuple types can be mapped by position.
- * POJO types require name equivalence to be mapped correctly as their fields do not have
- * an order.
- */
- def toDataSet[T](table: Table, typeInfo: TypeInformation[T]): DataSet[T] = {
- new JavaBatchTranslator(config).translate[T](table.relNode)(typeInfo)
- }
-
- /**
- * Registers a DataSet under a unique name, so that it can be used in SQL queries.
- * The fields of the DataSet type are used to name the Table fields.
- * @param name the Table name
- * @param dataset the DataSet to register
- */
- def registerDataSet[T](name: String, dataset: DataSet[T]): Unit = {
- registerDataSetInternal(name, dataset)
- }
-
- /**
- * Registers a DataSet under a unique name, so that it can be used in SQL queries.
- * The fields of the DataSet type are renamed to the given set of fields.
- *
- * @param name the Table name
- * @param dataset the DataSet to register
- * @param fields the Table field names
- */
- def registerDataSet[T](name: String, dataset: DataSet[T], fields: String): Unit = {
- val exprs = ExpressionParser
- .parseExpressionList(fields)
- .toArray
- registerDataSetInternal(name, dataset, exprs)
- }
-
- /**
- * Transforms the given DataStream to a [[org.apache.flink.api.table.Table]].
- * The fields of the DataStream type are renamed to the given set of fields:
- *
- * Example:
- *
- * {{{
- * tableEnv.fromDataStream(stream, "a, b")
- * }}}
- *
- * This will transform the stream containing elements of two fields to a table where the fields
- * are named a and b.
- */
- def fromDataStream[T](set: DataStream[T], fields: String): Table = {
- new JavaStreamTranslator(config).createTable(set, fields)
- }
-
- /**
- * Transforms the given DataStream to a [[org.apache.flink.api.table.Table]].
- * The fields of the DataStream type are used to name the
- * [[org.apache.flink.api.table.Table]] fields.
- */
- def fromDataStream[T](set: DataStream[T]): Table = {
- new JavaStreamTranslator(config).createTable(set)
- }
-
- /**
- * Converts the given [[org.apache.flink.api.table.Table]] to
- * a DataStream. The given type must have exactly the same field types and field order as the
- * [[org.apache.flink.api.table.Table]]. Row and tuple types can be mapped by position.
- * POJO types require name equivalence to be mapped correctly as their fields do not have
- * an order.
- */
- @SuppressWarnings(Array("unchecked"))
- def toDataStream[T](table: Table, clazz: Class[T]): DataStream[T] = {
- new JavaStreamTranslator(config).translate[T](table.relNode)(
- TypeExtractor.createTypeInfo(clazz).asInstanceOf[TypeInformation[T]])
- }
-
- /**
- * Converts the given [[org.apache.flink.api.table.Table]] to
- * a DataStream. The given type must have exactly the same field types and field order as the
- * [[org.apache.flink.api.table.Table]]. Row and tuple types can be mapped by position.
- * POJO types require name equivalence to be mapped correctly as their fields do not have
- * an order.
- */
- def toDataStream[T](table: Table, typeInfo: TypeInformation[T]): DataStream[T] = {
- new JavaStreamTranslator(config).translate[T](table.relNode)(typeInfo)
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/scala/table/BatchTableEnvironment.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/scala/table/BatchTableEnvironment.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/scala/table/BatchTableEnvironment.scala
new file mode 100644
index 0000000..a18f338
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/scala/table/BatchTableEnvironment.scala
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.api.scala.table
+
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.api.scala._
+import org.apache.flink.api.table.expressions.Expression
+import org.apache.flink.api.table.{TableConfig, Table}
+
+import scala.reflect.ClassTag
+
+/**
+ * The [[org.apache.flink.api.table.TableEnvironment]] for a Scala batch [[DataSet]]
+ * [[ExecutionEnvironment]].
+ *
+ * A TableEnvironment can be used to:
+ * - convert a [[DataSet]] to a [[Table]]
+ * - register a [[DataSet]] in the [[org.apache.flink.api.table.TableEnvironment]]'s catalog
+ * - register a [[Table]] in the [[org.apache.flink.api.table.TableEnvironment]]'s catalog
+ * - scan a registered table to obtain a [[Table]]
+ * - specify a SQL query on registered tables to obtain a [[Table]]
+ * - convert a [[Table]] into a [[DataSet]]
+ * - explain the AST and execution plan of a [[Table]]
+ *
+ * @param execEnv The Scala batch [[ExecutionEnvironment]] of the TableEnvironment.
+ * @param config The configuration of the TableEnvironment.
+ */
+class BatchTableEnvironment(
+ protected val execEnv: ExecutionEnvironment,
+ config: TableConfig)
+ extends org.apache.flink.api.table.BatchTableEnvironment(config) {
+
+ /**
+ * Converts the given [[DataSet]] into a [[Table]].
+ *
+ * The field names of the [[Table]] are automatically derived from the type of the [[DataSet]].
+ *
+ * @param dataSet The [[DataSet]] to be converted.
+ * @tparam T The type of the [[DataSet]].
+ * @return The converted [[Table]].
+ */
+ def fromDataSet[T](dataSet: DataSet[T]): Table = {
+
+ val name = createUniqueTableName()
+ registerDataSetInternal(name, dataSet.javaSet)
+ scan(name)
+ }
+
+ /**
+ * Converts the given [[DataSet]] into a [[Table]] with specified field names.
+ *
+ * Example:
+ *
+ * {{{
+ * val set: DataSet[(String, Long)] = ...
+ * val tab: Table = tableEnv.fromDataSet(set, 'a, 'b)
+ * }}}
+ *
+ * @param dataSet The [[DataSet]] to be converted.
+ * @param fields The field names of the resulting [[Table]].
+ * @tparam T The type of the [[DataSet]].
+ * @return The converted [[Table]].
+ */
+ def fromDataSet[T](dataSet: DataSet[T], fields: Expression*): Table = {
+
+ val name = createUniqueTableName()
+ registerDataSetInternal(name, dataSet.javaSet, fields.toArray)
+ scan(name)
+ }
+
+ /**
+ * Registers the given [[DataSet]] as table in the
+ * [[org.apache.flink.api.table.TableEnvironment]]'s catalog.
+ * Registered tables can be referenced in SQL queries.
+ *
+ * The field names of the [[Table]] are automatically derived from the type of the [[DataSet]].
+ *
+ * @param name The name under which the [[DataSet]] is registered in the catalog.
+ * @param dataSet The [[DataSet]] to register.
+ * @tparam T The type of the [[DataSet]] to register.
+ */
+ def registerDataSet[T](name: String, dataSet: DataSet[T]): Unit = {
+
+ checkValidTableName(name)
+ registerDataSetInternal(name, dataSet.javaSet)
+ }
+
+ /**
+ * Registers the given [[DataSet]] as table with specified field names in the
+ * [[org.apache.flink.api.table.TableEnvironment]]'s catalog.
+ * Registered tables can be referenced in SQL queries.
+ *
+ * Example:
+ *
+ * {{{
+ * val set: DataSet[(String, Long)] = ...
+ * tableEnv.registerDataSet("myTable", set, 'a, 'b)
+ * }}}
+ *
+ * @param name The name under which the [[DataSet]] is registered in the catalog.
+ * @param dataSet The [[DataSet]] to register.
+ * @param fields The field names of the registered table.
+ * @tparam T The type of the [[DataSet]] to register.
+ */
+ def registerDataSet[T](name: String, dataSet: DataSet[T], fields: Expression*): Unit = {
+
+ checkValidTableName(name)
+ registerDataSetInternal(name, dataSet.javaSet, fields.toArray)
+ }
+
+ /**
+ * Converts the given [[Table]] into a [[DataSet]] of a specified type.
+ *
+ * The fields of the [[Table]] are mapped to [[DataSet]] fields as follows:
+ * - [[org.apache.flink.api.table.Row]] and [[org.apache.flink.api.java.tuple.Tuple]]
+ * types: Fields are mapped by position, field types must match.
+ * - POJO [[DataSet]] types: Fields are mapped by field name, field types must match.
+ *
+ * @param table The [[Table]] to convert.
+ * @tparam T The type of the resulting [[DataSet]].
+ * @return The converted [[DataSet]].
+ */
+ def toDataSet[T: TypeInformation](table: Table): DataSet[T] = {
+ wrap[T](translate(table))(ClassTag.AnyRef.asInstanceOf[ClassTag[T]])
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/scala/table/DataSetConversions.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/scala/table/DataSetConversions.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/scala/table/DataSetConversions.scala
index 6e630ea..ce437c3 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/scala/table/DataSetConversions.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/scala/table/DataSetConversions.scala
@@ -18,56 +18,44 @@
package org.apache.flink.api.scala.table
import org.apache.flink.api.common.typeinfo.TypeInformation
-import org.apache.flink.api.table._
-import org.apache.flink.api.table.expressions.{UnresolvedFieldReference, Expression}
-import org.apache.flink.api.common.typeutils.CompositeType
-
import org.apache.flink.api.scala._
+import org.apache.flink.api.table._
+import org.apache.flink.api.table.expressions.Expression
/**
- * Methods for converting a [[DataSet]] to a [[Table]]. A [[DataSet]] is
- * wrapped in this by the implicit conversions in [[org.apache.flink.api.scala.table]].
- */
-class DataSetConversions[T](set: DataSet[T], inputType: TypeInformation[T]) {
+ * Holds methods to convert a [[DataSet]] into a [[Table]].
+ *
+ * @param dataSet The [[DataSet]] to convert.
+ * @param inputType The [[TypeInformation]] for the type of the [[DataSet]].
+ * @tparam T The type of the [[DataSet]].
+ */
+class DataSetConversions[T](dataSet: DataSet[T], inputType: TypeInformation[T]) {
/**
- * Converts the [[DataSet]] to a [[Table]]. The field names can be specified like this:
- *
- * {{{
- * val in: DataSet[(String, Int)] = ...
- * val table = in.as('a, 'b)
- * }}}
- *
- * This results in a [[Table]] that has field `a` of type `String` and field `b`
- * of type `Int`.
- */
- def as(fields: Expression*): Table = {
- new ScalaBatchTranslator().createTable(set, fields.toArray)
- }
-
- /**
- * Converts the [[DataSet]] to a [[Table]]. The field names will be taken from the field names
- * of the input type.
- *
- * Example:
- *
- * {{{
- * val in: DataSet[(String, Int)] = ...
- * val table = in.toTable
- * }}}
- *
- * Here, the result is a [[Table]] that has field `_1` of type `String` and field `_2`
- * of type `Int`.
- */
- def toTable: Table = {
-
- inputType match {
- case c: CompositeType[T] =>
- val resultFields = c.getFieldNames.map(UnresolvedFieldReference)
- as(resultFields: _*)
- case _ =>
- throw new IllegalArgumentException("" +
- "Please specify a field name with 'as' to convert an atomic type dataset to a table ")
+ * Converts the [[DataSet]] into a [[Table]].
+ *
+ * The field name of the new [[Table]] can be specified like this:
+ *
+ * {{{
+ * val env = ExecutionEnvironment.getExecutionEnvironment
+ * val tEnv = TableEnvironment.getTableEnvironment(env)
+ *
+ * val set: DataSet[(String, Int)] = ...
+ * val table = set.toTable(tEnv, 'name, 'amount)
+ * }}}
+ *
+ * If not explicitly specified, field names are automatically extracted from the type of
+ * the [[DataSet]].
+ *
+ * @param tableEnv The [[BatchTableEnvironment]] in which the new [[Table]] is created.
+ * @param fields The field names of the new [[Table]] (optional).
+ * @return The resulting [[Table]].
+ */
+ def toTable(tableEnv: BatchTableEnvironment, fields: Expression*): Table = {
+ if (fields.isEmpty) {
+ tableEnv.fromDataSet(dataSet)
+ } else {
+ tableEnv.fromDataSet(dataSet, fields: _*)
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/scala/table/DataStreamConversions.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/scala/table/DataStreamConversions.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/scala/table/DataStreamConversions.scala
index cd663b2..3b724cf 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/scala/table/DataStreamConversions.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/scala/table/DataStreamConversions.scala
@@ -17,50 +17,46 @@
*/
package org.apache.flink.api.scala.table
+import org.apache.flink.api.common.typeinfo.TypeInformation
import org.apache.flink.api.table._
-import org.apache.flink.api.table.expressions.{UnresolvedFieldReference, Expression}
-import org.apache.flink.api.common.typeutils.CompositeType
-import org.apache.flink.api.scala._
+import org.apache.flink.api.table.expressions.Expression
import org.apache.flink.streaming.api.scala.DataStream
/**
- * Methods for converting a [[DataStream]] to a [[Table]]. A [[DataStream]] is
- * wrapped in this by the implicit conversions in [[org.apache.flink.api.scala.table]].
- */
-class DataStreamConversions[T](set: DataStream[T], inputType: CompositeType[T]) {
-
- /**
- * Converts the [[DataStream]] to a [[Table]]. The field names can be specified like this:
- *
- * {{{
- * val in: DataStream[(String, Int)] = ...
- * val table = in.asStream('a, 'b)
- * }}}
- *
- * This results in a [[Table]] that has field `a` of type `String` and field `b`
- * of type `Int`.
- */
- def as(fields: Expression*): Table = {
- new ScalaStreamTranslator().createTable(set, fields.toArray)
- }
+ * Holds methods to convert a [[DataStream]] into a [[Table]].
+ *
+ * @param dataStream The [[DataStream]] to convert.
+ * @param inputType The [[TypeInformation]] for the type of the [[DataStream]].
+ * @tparam T The type of the [[DataStream]].
+ */
+class DataStreamConversions[T](dataStream: DataStream[T], inputType: TypeInformation[T]) {
/**
- * Converts the [[DataStream]] to a [[Table]]. The field names will be taken from the field names
- * of the input type.
- *
- * Example:
- *
- * {{{
- * val in: DataStream[(String, Int)] = ...
- * val table = in.toStreamTable
- * }}}
- *
- * Here, the result is a [[Table]] that has field `_1` of type `String` and field `_2`
- * of type `Int`.
- */
- def toStreamTable: Table = {
- val resultFields = inputType.getFieldNames.map(UnresolvedFieldReference)
- as(resultFields: _*)
+ * Converts the [[DataStream]] into a [[Table]].
+ *
+ * The field name of the new [[Table]] can be specified like this:
+ *
+ * {{{
+ * val env = StreamExecutionEnvironment.getExecutionEnvironment
+ * val tEnv = TableEnvironment.getTableEnvironment(env)
+ *
+ * val stream: DataStream[(String, Int)] = ...
+ * val table = stream.toTable(tEnv, 'name, 'amount)
+ * }}}
+ *
+ * If not explicitly specified, field names are automatically extracted from the type of
+ * the [[DataStream]].
+ *
+ * @param tableEnv The [[StreamTableEnvironment]] in which the new [[Table]] is created.
+ * @param fields The field names of the new [[Table]] (optional).
+ * @return The resulting [[Table]].
+ */
+ def toTable(tableEnv: StreamTableEnvironment, fields: Expression*): Table = {
+ if (fields.isEmpty) {
+ tableEnv.fromDataStream(dataStream)
+ } else {
+ tableEnv.fromDataStream(dataStream, fields:_*)
+ }
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/scala/table/ScalaBatchTranslator.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/scala/table/ScalaBatchTranslator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/scala/table/ScalaBatchTranslator.scala
deleted file mode 100644
index 642654a..0000000
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/scala/table/ScalaBatchTranslator.scala
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.api.scala.table
-
-import org.apache.calcite.rel.RelNode
-import org.apache.flink.api.java.table.JavaBatchTranslator
-import org.apache.flink.api.scala.wrap
-import org.apache.flink.api.table.plan._
-import org.apache.flink.api.table.{TableConfig, Table}
-import org.apache.flink.api.common.typeinfo.TypeInformation
-import org.apache.flink.api.scala.DataSet
-
-import scala.reflect.ClassTag
-
-/**
- * [[PlanTranslator]] for creating [[Table]]s from Scala [[DataSet]]s and
- * translating them back to Scala [[DataSet]]s.
- */
-class ScalaBatchTranslator(config: TableConfig = TableConfig.DEFAULT) extends PlanTranslator {
-
- private val javaTranslator = new JavaBatchTranslator(config)
-
- type Representation[A] = DataSet[A]
-
- override def createTable[A](
- repr: Representation[A],
- fieldIndexes: Array[Int],
- fieldNames: Array[String]): Table =
- {
- javaTranslator.createTable(repr.javaSet, fieldIndexes, fieldNames)
- }
-
- override def translate[O](op: RelNode)(implicit tpe: TypeInformation[O]): DataSet[O] = {
- // fake it till you make it ...
- wrap(javaTranslator.translate(op))(ClassTag.AnyRef.asInstanceOf[ClassTag[O]])
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/scala/table/ScalaStreamTranslator.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/scala/table/ScalaStreamTranslator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/scala/table/ScalaStreamTranslator.scala
deleted file mode 100644
index be3664f..0000000
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/scala/table/ScalaStreamTranslator.scala
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.api.scala.table
-
-import org.apache.calcite.rel.RelNode
-import org.apache.flink.api.java.table.JavaStreamTranslator
-import org.apache.flink.api.table.plan._
-import org.apache.flink.api.table.{TableConfig, Table}
-import org.apache.flink.api.common.typeinfo.TypeInformation
-import org.apache.flink.streaming.api.scala.DataStream
-import org.apache.flink.streaming.api.scala.asScalaStream
-
-/**
- * [[PlanTranslator]] for creating [[Table]]s from Scala [[DataStream]]s and
- * translating them back to Scala [[DataStream]]s.
- */
-class ScalaStreamTranslator(config: TableConfig = TableConfig.DEFAULT) extends PlanTranslator {
-
- private val javaTranslator = new JavaStreamTranslator(config)
-
- type Representation[A] = DataStream[A]
-
- override def createTable[A](
- repr: Representation[A],
- fieldIndexes: Array[Int],
- fieldNames: Array[String]): Table =
- {
- javaTranslator.createTable(repr.javaStream, fieldIndexes, fieldNames)
- }
-
- override def translate[O](op: RelNode)(implicit tpe: TypeInformation[O]): DataStream[O] = {
- asScalaStream(javaTranslator.translate(op))
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/scala/table/StreamTableEnvironment.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/scala/table/StreamTableEnvironment.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/scala/table/StreamTableEnvironment.scala
new file mode 100644
index 0000000..15ef55e
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/scala/table/StreamTableEnvironment.scala
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.api.scala.table
+
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.api.table.{TableConfig, Table}
+import org.apache.flink.api.table.expressions.Expression
+import org.apache.flink.streaming.api.scala.{StreamExecutionEnvironment, DataStream}
+
+import org.apache.flink.streaming.api.scala.asScalaStream
+
+/**
+ * The [[org.apache.flink.api.table.TableEnvironment]] for a Scala [[StreamExecutionEnvironment]].
+ *
+ * A TableEnvironment can be used to:
+ * - convert a [[DataStream]] to a [[Table]]
+ * - register a [[DataStream]] in the [[org.apache.flink.api.table.TableEnvironment]]'s catalog
+ * - register a [[Table]] in the [[org.apache.flink.api.table.TableEnvironment]]'s catalog
+ * - scan a registered table to obtain a [[Table]]
+ * - specify a SQL query on registered tables to obtain a [[Table]]
+ * - convert a [[Table]] into a [[DataStream]]
+ * - explain the AST and execution plan of a [[Table]]
+ *
+ * @param execEnv The Scala [[StreamExecutionEnvironment]] of the TableEnvironment.
+ * @param config The configuration of the TableEnvironment.
+ */
+class StreamTableEnvironment(
+ protected val execEnv: StreamExecutionEnvironment,
+ config: TableConfig)
+ extends org.apache.flink.api.table.StreamTableEnvironment(config) {
+
+ /**
+ * Converts the given [[DataStream]] into a [[Table]].
+ *
+ * The field names of the [[Table]] are automatically derived from the type of the
+ * [[DataStream]].
+ *
+ * @param dataStream The [[DataStream]] to be converted.
+ * @tparam T The type of the [[DataStream]].
+ * @return The converted [[Table]].
+ */
+ def fromDataStream[T](dataStream: DataStream[T]): Table = {
+
+ val name = createUniqueTableName()
+ registerDataStreamInternal(name, dataStream.javaStream)
+ ingest(name)
+ }
+
+ /**
+ * Converts the given [[DataStream]] into a [[Table]] with specified field names.
+ *
+ * Example:
+ *
+ * {{{
+ * val stream: DataStream[(String, Long)] = ...
+ * val tab: Table = tableEnv.fromDataStream(stream, 'a, 'b)
+ * }}}
+ *
+ * @param dataStream The [[DataStream]] to be converted.
+ * @param fields The field names of the resulting [[Table]].
+ * @tparam T The type of the [[DataStream]].
+ * @return The converted [[Table]].
+ */
+ def fromDataStream[T](dataStream: DataStream[T], fields: Expression*): Table = {
+
+ val name = createUniqueTableName()
+ registerDataStreamInternal(name, dataStream.javaStream, fields.toArray)
+ ingest(name)
+ }
+
+ /**
+ * Converts the given [[Table]] into a [[DataStream]] of a specified type.
+ *
+ * The fields of the [[Table]] are mapped to [[DataStream]] fields as follows:
+ * - [[org.apache.flink.api.table.Row]] and [[org.apache.flink.api.java.tuple.Tuple]]
+ * types: Fields are mapped by position, field types must match.
+ * - POJO [[DataStream]] types: Fields are mapped by field name, field types must match.
+ *
+ * @param table The [[Table]] to convert.
+ * @tparam T The type of the resulting [[DataStream]].
+ * @return The converted [[DataStream]].
+ */
+ def toDataStream[T: TypeInformation](table: Table): DataStream[T] = {
+ asScalaStream(translate(table))
+ }
+
+}
[06/12] flink git commit: [FLINK-3738] [table] Refactor
TableEnvironments. Remove Translators and TranslationContext.
Posted by fh...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/CastingITCase.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/CastingITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/CastingITCase.scala
index 5ca8c7f..c0499e4 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/CastingITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/CastingITCase.scala
@@ -23,7 +23,7 @@ import java.util.Date
import org.apache.flink.api.common.typeinfo.BasicTypeInfo
import org.apache.flink.api.scala._
import org.apache.flink.api.scala.table._
-import org.apache.flink.api.table.Row
+import org.apache.flink.api.table.{TableEnvironment, Row}
import org.apache.flink.api.table.codegen.CodeGenException
import org.apache.flink.test.util.MultipleProgramsTestBase.TestExecutionMode
import org.apache.flink.test.util.{MultipleProgramsTestBase, TestBaseUtils}
@@ -42,7 +42,9 @@ class CastingITCase(mode: TestExecutionMode) extends MultipleProgramsTestBase(mo
// don't test everything, just some common cast directions
val env = ExecutionEnvironment.getExecutionEnvironment
- val t = env.fromElements((1: Byte, 1: Short, 1, 1L, 1.0f, 1.0d, 1L, 1001.1)).toTable
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
+ val t = env.fromElements((1: Byte, 1: Short, 1, 1L, 1.0f, 1.0d, 1L, 1001.1)).toTable(tEnv)
.select('_1 + 1, '_2 + 1, '_3 + 1L, '_4 + 1.0f, '_5 + 1.0d, '_6 + 1, '_7 + 1.0d, '_8 + '_1)
val expected = "2,2,2,2.0,2.0,2.0,2.0,1002.1"
@@ -56,9 +58,11 @@ class CastingITCase(mode: TestExecutionMode) extends MultipleProgramsTestBase(mo
// don't test everything, just some common cast directions
val env = ExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
val t = env.fromElements(
(1: Byte, 1: Short, 1, 1L, 1.0f, 1.0d),
- (2: Byte, 2: Short, 2, 2L, 2.0f, 2.0d)).as('a, 'b, 'c, 'd, 'e, 'f)
+ (2: Byte, 2: Short, 2, 2L, 2.0f, 2.0d)).toTable(tEnv, 'a, 'b, 'c, 'd, 'e, 'f)
.filter('a > 1 && 'b > 1 && 'c > 1L && 'd > 1.0f && 'e > 1.0d && 'f > 1)
val expected = "2,2,2,2,2.0,2.0"
@@ -72,7 +76,9 @@ class CastingITCase(mode: TestExecutionMode) extends MultipleProgramsTestBase(mo
def testAutoCastToString(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val t = env.fromElements((1: Byte, 1: Short, 1, 1L, 1.0f, 1.0d, new Date(0))).toTable
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
+ val t = env.fromElements((1: Byte, 1: Short, 1, 1L, 1.0f, 1.0d, new Date(0))).toTable(tEnv)
.select('_1 + "b", '_2 + "s", '_3 + "i", '_4 + "L", '_5 + "f", '_6 + "d", '_7 + "Date")
val expected = "1b,1s,1i,1L,1.0f,1.0d,1970-01-01 00:00:00.000Date"
@@ -84,8 +90,10 @@ class CastingITCase(mode: TestExecutionMode) extends MultipleProgramsTestBase(mo
def testCasting(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
val t = env.fromElements((1, 0.0, 1L, true))
- .toTable
+ .toTable(tEnv)
.select(
// * -> String
'_1.cast(BasicTypeInfo.STRING_TYPE_INFO),
@@ -121,8 +129,10 @@ class CastingITCase(mode: TestExecutionMode) extends MultipleProgramsTestBase(mo
def testCastFromString(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
val t = env.fromElements(("1", "true", "2.0"))
- .toTable
+ .toTable(tEnv)
.select(
// String -> BASIC TYPE (not String, Date, Void, Character)
'_1.cast(BasicTypeInfo.BYTE_TYPE_INFO),
@@ -143,8 +153,10 @@ class CastingITCase(mode: TestExecutionMode) extends MultipleProgramsTestBase(mo
def testCastDateFromString(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
val t = env.fromElements(("2011-05-03", "15:51:36", "2011-05-03 15:51:36.000", "1446473775"))
- .toTable
+ .toTable(tEnv)
.select(
'_1.cast(BasicTypeInfo.DATE_TYPE_INFO).cast(BasicTypeInfo.STRING_TYPE_INFO),
'_2.cast(BasicTypeInfo.DATE_TYPE_INFO).cast(BasicTypeInfo.STRING_TYPE_INFO),
@@ -161,8 +173,10 @@ class CastingITCase(mode: TestExecutionMode) extends MultipleProgramsTestBase(mo
@Test
def testCastDateToStringAndLong(): Unit = {
val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
val ds = env.fromElements(("2011-05-03 15:51:36.000", "1304437896000"))
- val t = ds.toTable
+ val t = ds.toTable(tEnv)
.select('_1.cast(BasicTypeInfo.DATE_TYPE_INFO).as('f0),
'_2.cast(BasicTypeInfo.DATE_TYPE_INFO).as('f1))
.select('f0.cast(BasicTypeInfo.STRING_TYPE_INFO),
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/DistinctITCase.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/DistinctITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/DistinctITCase.scala
index 770819a..fc1284e 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/DistinctITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/DistinctITCase.scala
@@ -21,7 +21,7 @@ package org.apache.flink.api.scala.table.test
import org.apache.flink.api.scala._
import org.apache.flink.api.scala.table._
import org.apache.flink.api.scala.util.CollectionDataSets
-import org.apache.flink.api.table.Row
+import org.apache.flink.api.table.{TableEnvironment, Row}
import org.apache.flink.test.util.{TestBaseUtils, MultipleProgramsTestBase}
import org.apache.flink.test.util.MultipleProgramsTestBase.TestExecutionMode
import org.junit.Test
@@ -36,8 +36,9 @@ class DistinctITCase(mode: TestExecutionMode) extends MultipleProgramsTestBase(m
@Test
def testDistinct(): Unit = {
val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.get3TupleDataSet(env).as('a, 'b, 'c)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+ val ds = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv, 'a, 'b, 'c)
val distinct = ds.select('b).distinct()
val expected = "1\n" + "2\n" + "3\n" + "4\n" + "5\n" + "6\n"
@@ -48,8 +49,9 @@ class DistinctITCase(mode: TestExecutionMode) extends MultipleProgramsTestBase(m
@Test
def testDistinctAfterAggregate(): Unit = {
val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.get5TupleDataSet(env).as('a, 'b, 'c, 'd, 'e)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+ val ds = CollectionDataSets.get5TupleDataSet(env).toTable(tEnv, 'a, 'b, 'c, 'd, 'e)
val distinct = ds.groupBy('a, 'e).select('e).distinct()
val expected = "1\n" + "2\n" + "3\n"
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/ExpressionsITCase.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/ExpressionsITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/ExpressionsITCase.scala
index 29b3be4..9e908dc 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/ExpressionsITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/ExpressionsITCase.scala
@@ -23,8 +23,9 @@ import java.util.Date
import org.apache.flink.api.common.typeinfo.BasicTypeInfo
import org.apache.flink.api.scala._
import org.apache.flink.api.scala.table._
-import org.apache.flink.api.table.Row
-import org.apache.flink.api.table.expressions.{Literal, Null}
+import org.apache.flink.api.table.expressions.Null
+import org.apache.flink.api.table.{TableEnvironment, Row}
+import org.apache.flink.api.table.expressions.Literal
import org.apache.flink.api.table.test.utils.TableProgramsTestBase
import org.apache.flink.api.table.test.utils.TableProgramsTestBase.TableConfigMode
import org.apache.flink.test.util.MultipleProgramsTestBase.TestExecutionMode
@@ -38,18 +39,20 @@ import scala.collection.JavaConverters._
@RunWith(classOf[Parameterized])
class ExpressionsITCase(
mode: TestExecutionMode,
- config: TableConfigMode)
- extends TableProgramsTestBase(mode, config) {
+ configMode: TableConfigMode)
+ extends TableProgramsTestBase(mode, configMode) {
@Test
def testArithmetic(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val t = env.fromElements((5, 10)).as('a, 'b)
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
+
+ val t = env.fromElements((5, 10)).toTable(tEnv, 'a, 'b)
.select('a - 5, 'a + 5, 'a / 2, 'a * 2, 'a % 2, -'a)
val expected = "0,10,2,10,1,-5"
- val results = t.toDataSet[Row](getConfig).collect()
+ val results = t.toDataSet[Row].collect()
TestBaseUtils.compareResultAsText(results.asJava, expected)
}
@@ -57,11 +60,13 @@ class ExpressionsITCase(
def testLogic(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val t = env.fromElements((5, true)).as('a, 'b)
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
+
+ val t = env.fromElements((5, true)).toTable(tEnv, 'a, 'b)
.select('b && true, 'b && false, 'b || false, !'b)
val expected = "true,false,true,false"
- val results = t.toDataSet[Row](getConfig).collect()
+ val results = t.toDataSet[Row].collect()
TestBaseUtils.compareResultAsText(results.asJava, expected)
}
@@ -69,11 +74,13 @@ class ExpressionsITCase(
def testComparisons(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val t = env.fromElements((5, 5, 4)).as('a, 'b, 'c)
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
+
+ val t = env.fromElements((5, 5, 4)).toTable(tEnv, 'a, 'b, 'c)
.select('a > 'c, 'a >= 'b, 'a < 'c, 'a.isNull, 'a.isNotNull)
val expected = "true,true,false,false,true"
- val results = t.toDataSet[Row](getConfig).collect()
+ val results = t.toDataSet[Row].collect()
TestBaseUtils.compareResultAsText(results.asJava, expected)
}
@@ -81,32 +88,34 @@ class ExpressionsITCase(
def testCaseInsensitiveForAs(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
- val t = env.fromElements((3, 5.toByte)).as('a, 'b)
+ val t = env.fromElements((3, 5.toByte)).toTable(tEnv, 'a, 'b)
.groupBy("a").select("a, a.count As cnt")
val expected = "3,1"
- val results = t.toDataSet[Row](getConfig).collect()
+ val results = t.toDataSet[Row].collect()
TestBaseUtils.compareResultAsText(results.asJava, expected)
}
@Test
def testNullLiteral(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
- val t = env.fromElements((1, 0)).as('a, 'b)
+ val t = env.fromElements((1, 0)).toTable(tEnv, 'a, 'b)
.select(
'a,
'b,
Null(BasicTypeInfo.INT_TYPE_INFO),
Null(BasicTypeInfo.STRING_TYPE_INFO) === "")
- val expected = if (getConfig.getNullCheck) {
+ val expected = if (config.getNullCheck) {
"1,0,null,null"
} else {
"1,0,-1,true"
}
- val results = t.toDataSet[Row](getConfig).collect()
+ val results = t.toDataSet[Row].collect()
TestBaseUtils.compareResultAsText(results.asJava, expected)
}
@@ -115,14 +124,15 @@ class ExpressionsITCase(
@Test
def testDateLiteral(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
- val t = env.fromElements((0L, "test")).as('a, 'b)
+ val t = env.fromElements((0L, "test")).toTable(tEnv, 'a, 'b)
.select('a,
Literal(new Date(0)).cast(BasicTypeInfo.STRING_TYPE_INFO),
'a.cast(BasicTypeInfo.DATE_TYPE_INFO).cast(BasicTypeInfo.STRING_TYPE_INFO))
val expected = "0,1970-01-01 00:00:00.000,1970-01-01 00:00:00.000"
- val results = t.toDataSet[Row](getConfig).collect()
+ val results = t.toDataSet[Row].collect()
TestBaseUtils.compareResultAsText(results.asJava, expected)
}
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/FilterITCase.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/FilterITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/FilterITCase.scala
index 946f584..51dfe74 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/FilterITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/FilterITCase.scala
@@ -21,7 +21,7 @@ package org.apache.flink.api.scala.table.test
import org.apache.flink.api.scala._
import org.apache.flink.api.scala.table._
import org.apache.flink.api.scala.util.CollectionDataSets
-import org.apache.flink.api.table.Row
+import org.apache.flink.api.table.{TableEnvironment, Row}
import org.apache.flink.api.table.expressions.Literal
import org.apache.flink.api.table.test.utils.TableProgramsTestBase
import TableProgramsTestBase.TableConfigMode
@@ -43,19 +43,23 @@ class FilterITCase(
@Test
def testAllRejectingFilter(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.get3TupleDataSet(env).as('a, 'b, 'c)
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
+
+ val ds = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv, 'a, 'b, 'c)
val filterDs = ds.filter( Literal(false) )
val expected = "\n"
- val results = filterDs.toDataSet[Row](getConfig).collect()
+ val results = filterDs.toDataSet[Row].collect()
TestBaseUtils.compareResultAsText(results.asJava, expected)
}
@Test
def testAllPassingFilter(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.get3TupleDataSet(env).as('a, 'b, 'c)
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
+
+ val ds = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv, 'a, 'b, 'c)
val filterDs = ds.filter( Literal(true) )
val expected = "1,1,Hi\n" + "2,2,Hello\n" + "3,2,Hello world\n" + "4,3,Hello world, " +
@@ -64,25 +68,29 @@ class FilterITCase(
"Comment#5\n" + "12,5,Comment#6\n" + "13,5,Comment#7\n" + "14,5,Comment#8\n" + "15,5," +
"Comment#9\n" + "16,6,Comment#10\n" + "17,6,Comment#11\n" + "18,6,Comment#12\n" + "19," +
"6,Comment#13\n" + "20,6,Comment#14\n" + "21,6,Comment#15\n"
- val results = filterDs.toDataSet[Row](getConfig).collect()
+ val results = filterDs.toDataSet[Row].collect()
TestBaseUtils.compareResultAsText(results.asJava, expected)
}
@Test
def testFilterOnStringTupleField(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.get3TupleDataSet(env).as('a, 'b, 'c)
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
+
+ val ds = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv, 'a, 'b, 'c)
val filterDs = ds.filter( 'c.like("%world%") )
val expected = "3,2,Hello world\n" + "4,3,Hello world, how are you?\n"
- val results = filterDs.toDataSet[Row](getConfig).collect()
+ val results = filterDs.toDataSet[Row].collect()
TestBaseUtils.compareResultAsText(results.asJava, expected)
}
@Test
def testFilterOnIntegerTupleField(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.get3TupleDataSet(env).as('a, 'b, 'c)
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
+
+ val ds = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv, 'a, 'b, 'c)
val filterDs = ds.filter( 'a % 2 === 0 )
@@ -90,14 +98,16 @@ class FilterITCase(
"6,3,Luke Skywalker\n" + "8,4," + "Comment#2\n" + "10,4,Comment#4\n" +
"12,5,Comment#6\n" + "14,5,Comment#8\n" + "16,6," +
"Comment#10\n" + "18,6,Comment#12\n" + "20,6,Comment#14\n"
- val results = filterDs.toDataSet[Row](getConfig).collect()
+ val results = filterDs.toDataSet[Row].collect()
TestBaseUtils.compareResultAsText(results.asJava, expected)
}
@Test
def testNotEquals(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.get3TupleDataSet(env).as('a, 'b, 'c)
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
+
+ val ds = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv, 'a, 'b, 'c)
val filterDs = ds.filter( 'a % 2 !== 0)
val expected = "1,1,Hi\n" + "3,2,Hello world\n" +
@@ -111,7 +121,9 @@ class FilterITCase(
@Test
def testDisjunctivePredicate(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.get3TupleDataSet(env).as('a, 'b, 'c)
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
+
+ val ds = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv, 'a, 'b, 'c)
val filterDs = ds.filter( 'a < 2 || 'a > 20)
val expected = "1,1,Hi\n" + "21,6,Comment#15\n"
@@ -122,7 +134,9 @@ class FilterITCase(
@Test
def testConsecutiveFilters(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.get3TupleDataSet(env).as('a, 'b, 'c)
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
+
+ val ds = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv, 'a, 'b, 'c)
val filterDs = ds.filter('a % 2 !== 0).filter('b % 2 === 0)
val expected = "3,2,Hello world\n" + "7,4,Comment#1\n" +
@@ -135,31 +149,37 @@ class FilterITCase(
@Test
def testFilterBasicType(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
+
val ds = CollectionDataSets.getStringDataSet(env)
- val filterDs = ds.as('a).filter( 'a.like("H%") )
+ val filterDs = ds.toTable(tEnv, 'a).filter( 'a.like("H%") )
val expected = "Hi\n" + "Hello\n" + "Hello world\n" + "Hello world, how are you?\n"
- val results = filterDs.toDataSet[Row](getConfig).collect()
+ val results = filterDs.toDataSet[Row].collect()
TestBaseUtils.compareResultAsText(results.asJava, expected)
}
@Test
def testFilterOnCustomType(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
+
val ds = CollectionDataSets.getCustomTypeDataSet(env)
- val filterDs = ds.as('myInt as 'i, 'myLong as 'l, 'myString as 's)
+ val filterDs = ds.toTable(tEnv, 'myInt as 'i, 'myLong as 'l, 'myString as 's)
.filter( 's.like("%a%") )
val expected = "3,3,Hello world, how are you?\n" + "3,4,I am fine.\n" + "3,5,Luke Skywalker\n"
- val results = filterDs.toDataSet[Row](getConfig).collect()
+ val results = filterDs.toDataSet[Row].collect()
TestBaseUtils.compareResultAsText(results.asJava, expected)
}
@Test(expected = classOf[IllegalArgumentException])
def testFilterInvalidFieldName(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.get3TupleDataSet(env).as('a, 'b, 'c)
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
+
+ val ds = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv, 'a, 'b, 'c)
// must fail. Field 'foo does not exist
ds.filter( 'foo === 2 )
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/GroupedAggregationsITCase.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/GroupedAggregationsITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/GroupedAggregationsITCase.scala
index 01aa00f..a9edbb0 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/GroupedAggregationsITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/GroupedAggregationsITCase.scala
@@ -18,7 +18,7 @@
package org.apache.flink.api.scala.table.test
-import org.apache.flink.api.table.Row
+import org.apache.flink.api.table.{TableEnvironment, Row}
import org.apache.flink.api.scala._
import org.apache.flink.api.scala.table._
import org.apache.flink.api.scala.util.CollectionDataSets
@@ -38,7 +38,9 @@ class GroupedAggregationsITCase(mode: TestExecutionMode) extends MultipleProgram
def testGroupingOnNonExistentField(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val t = CollectionDataSets.get3TupleDataSet(env).as('a, 'b, 'c)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
+ val t = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv, 'a, 'b, 'c)
// must fail. '_foo not a valid field
.groupBy('_foo)
.select('a.avg)
@@ -48,7 +50,9 @@ class GroupedAggregationsITCase(mode: TestExecutionMode) extends MultipleProgram
def testGroupingInvalidSelection(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val t = CollectionDataSets.get3TupleDataSet(env).as('a, 'b, 'c)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
+ val t = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv, 'a, 'b, 'c)
.groupBy('a, 'b)
// must fail. 'c is not a grouping key or aggregation
.select('c)
@@ -58,7 +62,9 @@ class GroupedAggregationsITCase(mode: TestExecutionMode) extends MultipleProgram
def testGroupedAggregate(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val t = CollectionDataSets.get3TupleDataSet(env).as('a, 'b, 'c)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
+ val t = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv, 'a, 'b, 'c)
.groupBy('b)
.select('b, 'a.sum)
@@ -71,7 +77,9 @@ class GroupedAggregationsITCase(mode: TestExecutionMode) extends MultipleProgram
def testGroupingKeyForwardIfNotUsed(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val t = CollectionDataSets.get3TupleDataSet(env).as('a, 'b, 'c)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
+ val t = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv, 'a, 'b, 'c)
.groupBy('b)
.select('a.sum)
@@ -84,8 +92,10 @@ class GroupedAggregationsITCase(mode: TestExecutionMode) extends MultipleProgram
def testGroupNoAggregation(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
val t = CollectionDataSets.get3TupleDataSet(env)
- .as('a, 'b, 'c)
+ .toTable(tEnv, 'a, 'b, 'c)
.groupBy('b)
.select('a.sum as 'd, 'b)
.groupBy('b, 'd)
@@ -101,6 +111,8 @@ class GroupedAggregationsITCase(mode: TestExecutionMode) extends MultipleProgram
// This uses very long keys to force serialized comparison.
// With short keys, the normalized key is sufficient.
val env = ExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
val ds = env.fromElements(
("hhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhaa", 1, 2),
("hhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhaa", 1, 2),
@@ -111,7 +123,7 @@ class GroupedAggregationsITCase(mode: TestExecutionMode) extends MultipleProgram
("hhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhab", 1, 2),
("hhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhab", 1, 2),
("hhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhab", 1, 2))
- .rebalance().setParallelism(2).as('a, 'b, 'c)
+ .rebalance().setParallelism(2).toTable(tEnv, 'a, 'b, 'c)
.groupBy('a, 'b)
.select('c.sum)
@@ -124,7 +136,9 @@ class GroupedAggregationsITCase(mode: TestExecutionMode) extends MultipleProgram
def testGroupedAggregateWithConstant1(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val t = CollectionDataSets.get3TupleDataSet(env).as('a, 'b, 'c)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
+ val t = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv, 'a, 'b, 'c)
.select('a, 4 as 'four, 'b)
.groupBy('four, 'a)
.select('four, 'b.sum)
@@ -141,7 +155,9 @@ class GroupedAggregationsITCase(mode: TestExecutionMode) extends MultipleProgram
def testGroupedAggregateWithConstant2(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val t = CollectionDataSets.get3TupleDataSet(env).as('a, 'b, 'c)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
+ val t = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv, 'a, 'b, 'c)
.select('b, 4 as 'four, 'a)
.groupBy('b, 'four)
.select('four, 'a.sum)
@@ -155,7 +171,9 @@ class GroupedAggregationsITCase(mode: TestExecutionMode) extends MultipleProgram
def testGroupedAggregateWithExpression(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val t = CollectionDataSets.get5TupleDataSet(env).as('a, 'b, 'c, 'd, 'e)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
+ val t = CollectionDataSets.get5TupleDataSet(env).toTable(tEnv, 'a, 'b, 'c, 'd, 'e)
.groupBy('e, 'b % 3)
.select('c.min, 'e, 'a.avg, 'd.count)
@@ -169,7 +187,9 @@ class GroupedAggregationsITCase(mode: TestExecutionMode) extends MultipleProgram
def testGroupedAggregateWithFilter(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val t = CollectionDataSets.get3TupleDataSet(env).as('a, 'b, 'c)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
+ val t = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv, 'a, 'b, 'c)
.groupBy('b)
.select('b, 'a.sum)
.where('b === 2)
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/JoinITCase.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/JoinITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/JoinITCase.scala
index 41983bc..2442091 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/JoinITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/JoinITCase.scala
@@ -18,7 +18,7 @@
package org.apache.flink.api.scala.table.test
-import org.apache.flink.api.table.{TableException, Row}
+import org.apache.flink.api.table.{TableEnvironment, TableException, Row}
import org.apache.flink.api.scala._
import org.apache.flink.api.scala.table._
import org.apache.flink.api.scala.util.CollectionDataSets
@@ -36,8 +36,10 @@ class JoinITCase(mode: TestExecutionMode) extends MultipleProgramsTestBase(mode)
@Test
def testJoin(): Unit = {
val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = CollectionDataSets.getSmall3TupleDataSet(env).as('a, 'b, 'c)
- val ds2 = CollectionDataSets.get5TupleDataSet(env).as('d, 'e, 'f, 'g, 'h)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
+ val ds1 = CollectionDataSets.getSmall3TupleDataSet(env).toTable(tEnv, 'a, 'b, 'c)
+ val ds2 = CollectionDataSets.get5TupleDataSet(env).toTable(tEnv, 'd, 'e, 'f, 'g, 'h)
val joinT = ds1.join(ds2).where('b === 'e).select('c, 'g)
@@ -50,8 +52,10 @@ class JoinITCase(mode: TestExecutionMode) extends MultipleProgramsTestBase(mode)
def testJoinWithFilter(): Unit = {
val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = CollectionDataSets.getSmall3TupleDataSet(env).as('a, 'b, 'c)
- val ds2 = CollectionDataSets.get5TupleDataSet(env).as('d, 'e, 'f, 'g, 'h)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
+ val ds1 = CollectionDataSets.getSmall3TupleDataSet(env).toTable(tEnv, 'a, 'b, 'c)
+ val ds2 = CollectionDataSets.get5TupleDataSet(env).toTable(tEnv, 'd, 'e, 'f, 'g, 'h)
val joinT = ds1.join(ds2).where('b === 'e && 'b < 2).select('c, 'g)
@@ -63,8 +67,10 @@ class JoinITCase(mode: TestExecutionMode) extends MultipleProgramsTestBase(mode)
@Test
def testJoinWithJoinFilter(): Unit = {
val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = CollectionDataSets.get3TupleDataSet(env).as('a, 'b, 'c)
- val ds2 = CollectionDataSets.get5TupleDataSet(env).as('d, 'e, 'f, 'g, 'h)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
+ val ds1 = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv, 'a, 'b, 'c)
+ val ds2 = CollectionDataSets.get5TupleDataSet(env).toTable(tEnv, 'd, 'e, 'f, 'g, 'h)
val joinT = ds1.join(ds2).where('b === 'e && 'a < 6 && 'h < 'b).select('c, 'g)
@@ -77,8 +83,10 @@ class JoinITCase(mode: TestExecutionMode) extends MultipleProgramsTestBase(mode)
@Test
def testJoinWithMultipleKeys(): Unit = {
val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = CollectionDataSets.get3TupleDataSet(env).as('a, 'b, 'c)
- val ds2 = CollectionDataSets.get5TupleDataSet(env).as('d, 'e, 'f, 'g, 'h)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
+ val ds1 = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv, 'a, 'b, 'c)
+ val ds2 = CollectionDataSets.get5TupleDataSet(env).toTable(tEnv, 'd, 'e, 'f, 'g, 'h)
val joinT = ds1.join(ds2).filter('a === 'd && 'b === 'h).select('c, 'g)
@@ -91,8 +99,10 @@ class JoinITCase(mode: TestExecutionMode) extends MultipleProgramsTestBase(mode)
@Test(expected = classOf[IllegalArgumentException])
def testJoinNonExistingKey(): Unit = {
val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = CollectionDataSets.getSmall3TupleDataSet(env).as('a, 'b, 'c)
- val ds2 = CollectionDataSets.get5TupleDataSet(env).as('d, 'e, 'f, 'g, 'h)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
+ val ds1 = CollectionDataSets.getSmall3TupleDataSet(env).toTable(tEnv, 'a, 'b, 'c)
+ val ds2 = CollectionDataSets.get5TupleDataSet(env).toTable(tEnv, 'd, 'e, 'f, 'g, 'h)
ds1.join(ds2)
// must fail. Field 'foo does not exist
@@ -103,8 +113,10 @@ class JoinITCase(mode: TestExecutionMode) extends MultipleProgramsTestBase(mode)
@Test(expected = classOf[TableException])
def testJoinWithNonMatchingKeyTypes(): Unit = {
val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = CollectionDataSets.getSmall3TupleDataSet(env).as('a, 'b, 'c)
- val ds2 = CollectionDataSets.get5TupleDataSet(env).as('d, 'e, 'f, 'g, 'h)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
+ val ds1 = CollectionDataSets.getSmall3TupleDataSet(env).toTable(tEnv, 'a, 'b, 'c)
+ val ds2 = CollectionDataSets.get5TupleDataSet(env).toTable(tEnv, 'd, 'e, 'f, 'g, 'h)
ds1.join(ds2)
// must fail. Field 'a is Int, and 'g is String
@@ -115,8 +127,10 @@ class JoinITCase(mode: TestExecutionMode) extends MultipleProgramsTestBase(mode)
@Test(expected = classOf[IllegalArgumentException])
def testJoinWithAmbiguousFields(): Unit = {
val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = CollectionDataSets.getSmall3TupleDataSet(env).as('a, 'b, 'c)
- val ds2 = CollectionDataSets.get5TupleDataSet(env).as('d, 'e, 'f, 'g, 'c)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
+ val ds1 = CollectionDataSets.getSmall3TupleDataSet(env).toTable(tEnv, 'a, 'b, 'c)
+ val ds2 = CollectionDataSets.get5TupleDataSet(env).toTable(tEnv, 'd, 'e, 'f, 'g, 'c)
ds1.join(ds2)
// must fail. Both inputs share the same field 'c
@@ -127,8 +141,10 @@ class JoinITCase(mode: TestExecutionMode) extends MultipleProgramsTestBase(mode)
@Test(expected = classOf[TableException])
def testNoEqualityJoinPredicate1(): Unit = {
val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = CollectionDataSets.getSmall3TupleDataSet(env).as('a, 'b, 'c)
- val ds2 = CollectionDataSets.get5TupleDataSet(env).as('d, 'e, 'f, 'g, 'h)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
+ val ds1 = CollectionDataSets.getSmall3TupleDataSet(env).toTable(tEnv, 'a, 'b, 'c)
+ val ds2 = CollectionDataSets.get5TupleDataSet(env).toTable(tEnv, 'd, 'e, 'f, 'g, 'h)
ds1.join(ds2)
// must fail. No equality join predicate
@@ -139,8 +155,10 @@ class JoinITCase(mode: TestExecutionMode) extends MultipleProgramsTestBase(mode)
@Test(expected = classOf[TableException])
def testNoEqualityJoinPredicate2(): Unit = {
val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = CollectionDataSets.getSmall3TupleDataSet(env).as('a, 'b, 'c)
- val ds2 = CollectionDataSets.get5TupleDataSet(env).as('d, 'e, 'f, 'g, 'h)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
+ val ds1 = CollectionDataSets.getSmall3TupleDataSet(env).toTable(tEnv, 'a, 'b, 'c)
+ val ds2 = CollectionDataSets.get5TupleDataSet(env).toTable(tEnv, 'd, 'e, 'f, 'g, 'h)
ds1.join(ds2)
// must fail. No equality join predicate
@@ -151,8 +169,10 @@ class JoinITCase(mode: TestExecutionMode) extends MultipleProgramsTestBase(mode)
@Test
def testJoinWithAggregation(): Unit = {
val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = CollectionDataSets.getSmall3TupleDataSet(env).as('a, 'b, 'c)
- val ds2 = CollectionDataSets.get5TupleDataSet(env).as('d, 'e, 'f, 'g, 'h)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
+ val ds1 = CollectionDataSets.getSmall3TupleDataSet(env).toTable(tEnv, 'a, 'b, 'c)
+ val ds2 = CollectionDataSets.get5TupleDataSet(env).toTable(tEnv, 'd, 'e, 'f, 'g, 'h)
val joinT = ds1.join(ds2).where('a === 'd).select('g.count)
@@ -163,10 +183,11 @@ class JoinITCase(mode: TestExecutionMode) extends MultipleProgramsTestBase(mode)
@Test
def testJoinWithGroupedAggregation(): Unit = {
-
val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = CollectionDataSets.getSmall3TupleDataSet(env).as('a, 'b, 'c)
- val ds2 = CollectionDataSets.get5TupleDataSet(env).as('d, 'e, 'f, 'g, 'h)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
+ val ds1 = CollectionDataSets.getSmall3TupleDataSet(env).toTable(tEnv, 'a, 'b, 'c)
+ val ds2 = CollectionDataSets.get5TupleDataSet(env).toTable(tEnv, 'd, 'e, 'f, 'g, 'h)
val joinT = ds1.join(ds2)
.where('a === 'd)
@@ -180,11 +201,12 @@ class JoinITCase(mode: TestExecutionMode) extends MultipleProgramsTestBase(mode)
@Test
def testJoinPushThroughJoin(): Unit = {
-
val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = CollectionDataSets.getSmall3TupleDataSet(env).as('a, 'b, 'c)
- val ds2 = CollectionDataSets.get5TupleDataSet(env).as('d, 'e, 'f, 'g, 'h)
- val ds3 = CollectionDataSets.getSmall3TupleDataSet(env).as('j, 'k, 'l)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
+ val ds1 = CollectionDataSets.getSmall3TupleDataSet(env).toTable(tEnv, 'a, 'b, 'c)
+ val ds2 = CollectionDataSets.get5TupleDataSet(env).toTable(tEnv, 'd, 'e, 'f, 'g, 'h)
+ val ds3 = CollectionDataSets.getSmall3TupleDataSet(env).toTable(tEnv, 'j, 'k, 'l)
val joinT = ds1.join(ds2)
.where(Literal(true))
@@ -199,10 +221,11 @@ class JoinITCase(mode: TestExecutionMode) extends MultipleProgramsTestBase(mode)
@Test
def testJoinWithDisjunctivePred(): Unit = {
-
val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = CollectionDataSets.get3TupleDataSet(env).as('a, 'b, 'c)
- val ds2 = CollectionDataSets.get5TupleDataSet(env).as('d, 'e, 'f, 'g, 'h)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
+ val ds1 = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv, 'a, 'b, 'c)
+ val ds2 = CollectionDataSets.get5TupleDataSet(env).toTable(tEnv, 'd, 'e, 'f, 'g, 'h)
val joinT = ds1.join(ds2).filter('a === 'd && ('b === 'e || 'b === 'e - 10 )).select('c, 'g)
@@ -216,10 +239,11 @@ class JoinITCase(mode: TestExecutionMode) extends MultipleProgramsTestBase(mode)
@Test
def testJoinWithExpressionPreds(): Unit = {
-
val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = CollectionDataSets.get3TupleDataSet(env).as('a, 'b, 'c)
- val ds2 = CollectionDataSets.get5TupleDataSet(env).as('d, 'e, 'f, 'g, 'h)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
+ val ds1 = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv, 'a, 'b, 'c)
+ val ds2 = CollectionDataSets.get5TupleDataSet(env).toTable(tEnv, 'd, 'e, 'f, 'g, 'h)
val joinT = ds1.join(ds2).filter('b === 'h + 1 && 'a - 1 === 'd + 2).select('c, 'g)
@@ -233,4 +257,17 @@ class JoinITCase(mode: TestExecutionMode) extends MultipleProgramsTestBase(mode)
TestBaseUtils.compareResultAsText(results.asJava, expected)
}
+ @Test(expected = classOf[TableException])
+ def testJoinTablesFromDifferentEnvs(): Unit = {
+ val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
+ val tEnv1 = TableEnvironment.getTableEnvironment(env)
+ val tEnv2 = TableEnvironment.getTableEnvironment(env)
+
+ val ds1 = CollectionDataSets.getSmall3TupleDataSet(env).toTable(tEnv1, 'a, 'b, 'c)
+ val ds2 = CollectionDataSets.get5TupleDataSet(env).toTable(tEnv2, 'd, 'e, 'f, 'g, 'h)
+
+ // Must fail. Tables are bound to different TableEnvironments.
+ ds1.join(ds2).where('b === 'e).select('c, 'g)
+ }
+
}
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/RegisterDataSetITCase.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/RegisterDataSetITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/RegisterDataSetITCase.scala
deleted file mode 100644
index 535c064..0000000
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/RegisterDataSetITCase.scala
+++ /dev/null
@@ -1,136 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.api.scala.table.test
-
-import org.apache.flink.api.scala._
-import org.apache.flink.api.scala.table._
-import org.apache.flink.api.scala.util.CollectionDataSets
-import org.apache.flink.api.table.{TableException, Row}
-import org.apache.flink.api.table.plan.TranslationContext
-import org.apache.flink.api.table.test.utils.TableProgramsTestBase
-import org.apache.flink.api.table.test.utils.TableProgramsTestBase.TableConfigMode
-import org.apache.flink.test.util.MultipleProgramsTestBase.TestExecutionMode
-import org.apache.flink.test.util.TestBaseUtils
-import org.junit._
-import org.junit.runner.RunWith
-import org.junit.runners.Parameterized
-
-import scala.collection.JavaConverters._
-
-@RunWith(classOf[Parameterized])
-class RegisterDataSetITCase(
- mode: TestExecutionMode,
- configMode: TableConfigMode)
- extends TableProgramsTestBase(mode, configMode) {
-
- @Test
- def testSimpleRegister(): Unit = {
-
- val tableName = "MyTable"
- val env = ExecutionEnvironment.getExecutionEnvironment
- val tEnv = getScalaTableEnvironment
- TranslationContext.reset()
-
- val ds = CollectionDataSets.get3TupleDataSet(env)
- tEnv.registerDataSet(tableName, ds)
- val t = tEnv.scan(tableName).select('_1, '_2, '_3)
-
- val expected = "1,1,Hi\n" + "2,2,Hello\n" + "3,2,Hello world\n" +
- "4,3,Hello world, how are you?\n" + "5,3,I am fine.\n" + "6,3,Luke Skywalker\n" +
- "7,4,Comment#1\n" + "8,4,Comment#2\n" + "9,4,Comment#3\n" + "10,4,Comment#4\n" +
- "11,5,Comment#5\n" + "12,5,Comment#6\n" + "13,5,Comment#7\n" + "14,5,Comment#8\n" +
- "15,5,Comment#9\n" + "16,6,Comment#10\n" + "17,6,Comment#11\n" + "18,6,Comment#12\n" +
- "19,6,Comment#13\n" + "20,6,Comment#14\n" + "21,6,Comment#15\n"
- val results = t.toDataSet[Row](getConfig).collect()
- TestBaseUtils.compareResultAsText(results.asJava, expected)
- }
-
- @Test
- def testRegisterWithFields(): Unit = {
-
- val tableName = "MyTable"
- val env = ExecutionEnvironment.getExecutionEnvironment
- val tEnv = getScalaTableEnvironment
- TranslationContext.reset()
-
- val ds = CollectionDataSets.get3TupleDataSet(env)
- tEnv.registerDataSet(tableName, ds, 'a, 'b, 'c)
- val t = tEnv.scan(tableName).select('a, 'b)
-
- val expected = "1,1\n" + "2,2\n" + "3,2\n" + "4,3\n" + "5,3\n" + "6,3\n" +
- "7,4\n" + "8,4\n" + "9,4\n" + "10,4\n" + "11,5\n" + "12,5\n" + "13,5\n" + "14,5\n" +
- "15,5\n" + "16,6\n" + "17,6\n" + "18,6\n" + "19,6\n" + "20,6\n" + "21,6\n"
- val results = t.toDataSet[Row](getConfig).collect()
- TestBaseUtils.compareResultAsText(results.asJava, expected)
- }
-
- @Test(expected = classOf[TableException])
- def testRegisterExistingDataSet(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val tEnv = getScalaTableEnvironment
- TranslationContext.reset()
-
- val ds1 = CollectionDataSets.get3TupleDataSet(env)
- tEnv.registerDataSet("MyTable", ds1)
- val ds2 = CollectionDataSets.get5TupleDataSet(env)
- tEnv.registerDataSet("MyTable", ds2)
- }
-
- @Test(expected = classOf[TableException])
- def testScanUnregisteredTable(): Unit = {
- val tEnv = getScalaTableEnvironment
- TranslationContext.reset()
-
- tEnv.scan("someTable")
- }
-
- @Test
- def testTableRegister(): Unit = {
-
- val tableName = "MyTable"
- val env = ExecutionEnvironment.getExecutionEnvironment
- val tEnv = getScalaTableEnvironment
- TranslationContext.reset()
-
- val t = CollectionDataSets.get3TupleDataSet(env).as('a, 'b, 'c)
- tEnv.registerTable(tableName, t)
-
- val regT = tEnv.scan(tableName).select('a, 'b).filter('a > 8)
-
- val expected = "9,4\n" + "10,4\n" +
- "11,5\n" + "12,5\n" + "13,5\n" + "14,5\n" +
- "15,5\n" + "16,6\n" + "17,6\n" + "18,6\n" +
- "19,6\n" + "20,6\n" + "21,6\n"
-
- val results = regT.toDataSet[Row](getConfig).collect()
- TestBaseUtils.compareResultAsText(results.asJava, expected)
- }
-
- @Test(expected = classOf[TableException])
- def testRegisterExistingTable(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val tEnv = getScalaTableEnvironment
- TranslationContext.reset()
-
- val t1 = CollectionDataSets.get3TupleDataSet(env).toTable
- tEnv.registerTable("MyTable", t1)
- val t2 = CollectionDataSets.get5TupleDataSet(env).toTable
- tEnv.registerDataSet("MyTable", t2)
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/SelectITCase.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/SelectITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/SelectITCase.scala
index 46d620a..82668a1 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/SelectITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/SelectITCase.scala
@@ -21,7 +21,7 @@ package org.apache.flink.api.scala.table.test
import org.apache.flink.api.scala._
import org.apache.flink.api.scala.table._
import org.apache.flink.api.scala.util.CollectionDataSets
-import org.apache.flink.api.table.Row
+import org.apache.flink.api.table.{TableEnvironment, Row}
import org.apache.flink.api.table.test.utils.TableProgramsTestBase
import TableProgramsTestBase.TableConfigMode
import org.apache.flink.api.table.test.utils.TableProgramsTestBase
@@ -41,9 +41,10 @@ class SelectITCase(
@Test
def testSimpleSelectAll(): Unit = {
-
val env = ExecutionEnvironment.getExecutionEnvironment
- val t = CollectionDataSets.get3TupleDataSet(env).toTable.select('_1, '_2, '_3)
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
+
+ val t = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv).select('_1, '_2, '_3)
val expected = "1,1,Hi\n" + "2,2,Hello\n" + "3,2,Hello world\n" +
"4,3,Hello world, how are you?\n" + "5,3,I am fine.\n" + "6,3,Luke Skywalker\n" +
@@ -51,15 +52,16 @@ class SelectITCase(
"11,5,Comment#5\n" + "12,5,Comment#6\n" + "13,5,Comment#7\n" + "14,5,Comment#8\n" +
"15,5,Comment#9\n" + "16,6,Comment#10\n" + "17,6,Comment#11\n" + "18,6,Comment#12\n" +
"19,6,Comment#13\n" + "20,6,Comment#14\n" + "21,6,Comment#15\n"
- val results = t.toDataSet[Row](getConfig).collect()
+ val results = t.toDataSet[Row].collect()
TestBaseUtils.compareResultAsText(results.asJava, expected)
}
@Test
def testSimpleSelectAllWithAs(): Unit = {
-
val env = ExecutionEnvironment.getExecutionEnvironment
- val t = CollectionDataSets.get3TupleDataSet(env).as('a, 'b, 'c).select('a, 'b, 'c)
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
+
+ val t = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv, 'a, 'b, 'c).select('a, 'b, 'c)
val expected = "1,1,Hi\n" + "2,2,Hello\n" + "3,2,Hello world\n" +
"4,3,Hello world, how are you?\n" + "5,3,I am fine.\n" + "6,3,Luke Skywalker\n" +
@@ -67,63 +69,68 @@ class SelectITCase(
"11,5,Comment#5\n" + "12,5,Comment#6\n" + "13,5,Comment#7\n" + "14,5,Comment#8\n" +
"15,5,Comment#9\n" + "16,6,Comment#10\n" + "17,6,Comment#11\n" + "18,6,Comment#12\n" +
"19,6,Comment#13\n" + "20,6,Comment#14\n" + "21,6,Comment#15\n"
- val results = t.toDataSet[Row](getConfig).collect()
+ val results = t.toDataSet[Row].collect()
TestBaseUtils.compareResultAsText(results.asJava, expected)
}
@Test
def testSimpleSelectWithNaming(): Unit = {
-
val env = ExecutionEnvironment.getExecutionEnvironment
- val t = CollectionDataSets.get3TupleDataSet(env).toTable
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
+
+ val t = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv)
.select('_1 as 'a, '_2 as 'b, '_1 as 'c)
.select('a, 'b)
val expected = "1,1\n" + "2,2\n" + "3,2\n" + "4,3\n" + "5,3\n" + "6,3\n" + "7,4\n" +
"8,4\n" + "9,4\n" + "10,4\n" + "11,5\n" + "12,5\n" + "13,5\n" + "14,5\n" + "15,5\n" +
"16,6\n" + "17,6\n" + "18,6\n" + "19,6\n" + "20,6\n" + "21,6\n"
- val results = t.toDataSet[Row](getConfig).collect()
+ val results = t.toDataSet[Row].collect()
TestBaseUtils.compareResultAsText(results.asJava, expected)
}
@Test
def testSimpleSelectRenameAll(): Unit = {
-
val env = ExecutionEnvironment.getExecutionEnvironment
- val t = CollectionDataSets.get3TupleDataSet(env).toTable
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
+
+ val t = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv)
.select('_1 as 'a, '_2 as 'b, '_3 as 'c)
.select('a, 'b)
val expected = "1,1\n" + "2,2\n" + "3,2\n" + "4,3\n" + "5,3\n" + "6,3\n" + "7,4\n" +
"8,4\n" + "9,4\n" + "10,4\n" + "11,5\n" + "12,5\n" + "13,5\n" + "14,5\n" + "15,5\n" +
"16,6\n" + "17,6\n" + "18,6\n" + "19,6\n" + "20,6\n" + "21,6\n"
- val results = t.toDataSet[Row](getConfig).collect()
+ val results = t.toDataSet[Row].collect()
TestBaseUtils.compareResultAsText(results.asJava, expected)
}
@Test(expected = classOf[IllegalArgumentException])
def testSelectInvalidFieldFields(): Unit = {
-
val env = ExecutionEnvironment.getExecutionEnvironment
- CollectionDataSets.get3TupleDataSet(env).as('a, 'b, 'c)
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
+
+ CollectionDataSets.get3TupleDataSet(env).toTable(tEnv, 'a, 'b, 'c)
// must fail. Field 'foo does not exist
.select('a, 'foo)
}
@Test(expected = classOf[IllegalArgumentException])
def testSelectAmbiguousRenaming(): Unit = {
-
val env = ExecutionEnvironment.getExecutionEnvironment
- CollectionDataSets.get3TupleDataSet(env).as('a, 'b, 'c)
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
+
+ CollectionDataSets.get3TupleDataSet(env).toTable(tEnv, 'a, 'b, 'c)
// must fail. 'a and 'b are both renamed to 'foo
.select('a + 1 as 'foo, 'b + 2 as 'foo).toDataSet[Row].print()
}
@Test(expected = classOf[IllegalArgumentException])
def testSelectAmbiguousRenaming2(): Unit = {
-
val env = ExecutionEnvironment.getExecutionEnvironment
- CollectionDataSets.get3TupleDataSet(env).as('a, 'b, 'c)
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
+
+ CollectionDataSets.get3TupleDataSet(env).toTable(tEnv, 'a, 'b, 'c)
// must fail. 'a and 'b are both renamed to 'a
.select('a, 'b as 'a).toDataSet[Row].print()
}
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/SqlExplainTest.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/SqlExplainTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/SqlExplainTest.scala
index 861a801..545721d 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/SqlExplainTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/SqlExplainTest.scala
@@ -20,7 +20,7 @@ package org.apache.flink.api.scala.table.test
import org.apache.flink.api.scala._
import org.apache.flink.api.scala.table._
-import org.apache.flink.api.table.plan.TranslationContext
+import org.apache.flink.api.table.TableEnvironment
import org.apache.flink.test.util.MultipleProgramsTestBase
import org.junit._
@@ -31,17 +31,16 @@ class SqlExplainTest
val testFilePath = SqlExplainTest.this.getClass.getResource("/").getFile
- @Before
- def resetContext(): Unit = {
- TranslationContext.reset()
- }
-
@Test
def testFilterWithoutExtended() : Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val table = env.fromElements((1, "hello")).as('a, 'b)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
- val result = table.filter("a % 2 = 0").explain()
+ val table = env.fromElements((1, "hello"))
+ .toTable(tEnv, 'a, 'b)
+ .filter("a % 2 = 0")
+
+ val result = tEnv.explain(table)
val source = scala.io.Source.fromFile(testFilePath +
"../../src/test/scala/resources/testFilter0.out").mkString
assertEquals(result, source)
@@ -50,9 +49,13 @@ class SqlExplainTest
@Test
def testFilterWithExtended() : Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val table = env.fromElements((1, "hello")).as('a, 'b)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
+ val table = env.fromElements((1, "hello"))
+ .toTable(tEnv, 'a, 'b)
+ .filter("a % 2 = 0")
- val result = table.filter("a % 2 = 0").explain(true)
+ val result = tEnv.explain(table, true)
val source = scala.io.Source.fromFile(testFilePath +
"../../src/test/scala/resources/testFilter1.out").mkString
assertEquals(result, source)
@@ -61,10 +64,13 @@ class SqlExplainTest
@Test
def testJoinWithoutExtended() : Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val table1 = env.fromElements((1, "hello")).as('a, 'b)
- val table2 = env.fromElements((1, "hello")).as('c, 'd)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
- val result = table1.join(table2).where("b = d").select("a, c").explain()
+ val table1 = env.fromElements((1, "hello")).toTable(tEnv, 'a, 'b)
+ val table2 = env.fromElements((1, "hello")).toTable(tEnv, 'c, 'd)
+ val table = table1.join(table2).where("b = d").select("a, c")
+
+ val result = tEnv.explain(table)
val source = scala.io.Source.fromFile(testFilePath +
"../../src/test/scala/resources/testJoin0.out").mkString
assertEquals(result, source)
@@ -73,10 +79,13 @@ class SqlExplainTest
@Test
def testJoinWithExtended() : Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val table1 = env.fromElements((1, "hello")).as('a, 'b)
- val table2 = env.fromElements((1, "hello")).as('c, 'd)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
+ val table1 = env.fromElements((1, "hello")).toTable(tEnv, 'a, 'b)
+ val table2 = env.fromElements((1, "hello")).toTable(tEnv, 'c, 'd)
+ val table = table1.join(table2).where("b = d").select("a, c")
- val result = table1.join(table2).where("b = d").select("a, c").explain(true)
+ val result = tEnv.explain(table, true)
val source = scala.io.Source.fromFile(testFilePath +
"../../src/test/scala/resources/testJoin1.out").mkString
assertEquals(result, source)
@@ -85,10 +94,13 @@ class SqlExplainTest
@Test
def testUnionWithoutExtended() : Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val table1 = env.fromElements((1, "hello")).as('count, 'word)
- val table2 = env.fromElements((1, "hello")).as('count, 'word)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
- val result = table1.unionAll(table2).explain()
+ val table1 = env.fromElements((1, "hello")).toTable(tEnv, 'count, 'word)
+ val table2 = env.fromElements((1, "hello")).toTable(tEnv, 'count, 'word)
+ val table = table1.unionAll(table2)
+
+ val result = tEnv.explain(table)
val source = scala.io.Source.fromFile(testFilePath +
"../../src/test/scala/resources/testUnion0.out").mkString
assertEquals(result, source)
@@ -97,10 +109,13 @@ class SqlExplainTest
@Test
def testUnionWithExtended() : Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val table1 = env.fromElements((1, "hello")).as('count, 'word)
- val table2 = env.fromElements((1, "hello")).as('count, 'word)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
+ val table1 = env.fromElements((1, "hello")).toTable(tEnv, 'count, 'word)
+ val table2 = env.fromElements((1, "hello")).toTable(tEnv, 'count, 'word)
+ val table = table1.unionAll(table2)
- val result = table1.unionAll(table2).explain(true)
+ val result = tEnv.explain(table, true)
val source = scala.io.Source.fromFile(testFilePath +
"../../src/test/scala/resources/testUnion1.out").mkString
assertEquals(result, source)
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/StringExpressionsITCase.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/StringExpressionsITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/StringExpressionsITCase.scala
index 310d133..1ad57b4 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/StringExpressionsITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/StringExpressionsITCase.scala
@@ -20,7 +20,7 @@ package org.apache.flink.api.scala.table.test
import org.apache.flink.api.scala._
import org.apache.flink.api.scala.table._
-import org.apache.flink.api.table.Row
+import org.apache.flink.api.table.{TableEnvironment, Row}
import org.apache.flink.api.table.codegen.CodeGenException
import org.apache.flink.test.util.MultipleProgramsTestBase.TestExecutionMode
import org.apache.flink.test.util.{MultipleProgramsTestBase, TestBaseUtils}
@@ -36,7 +36,9 @@ class StringExpressionsITCase(mode: TestExecutionMode) extends MultipleProgramsT
@Test
def testSubstring(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val t = env.fromElements(("AAAA", 2), ("BBBB", 1)).as('a, 'b)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
+ val t = env.fromElements(("AAAA", 2), ("BBBB", 1)).toTable(tEnv, 'a, 'b)
.select('a.substring(1, 'b))
val expected = "AA\nB"
@@ -47,7 +49,9 @@ class StringExpressionsITCase(mode: TestExecutionMode) extends MultipleProgramsT
@Test
def testSubstringWithMaxEnd(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val t = env.fromElements(("ABCD", 3), ("ABCD", 2)).as('a, 'b)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
+ val t = env.fromElements(("ABCD", 3), ("ABCD", 2)).toTable(tEnv, 'a, 'b)
.select('a.substring('b))
val expected = "CD\nBCD"
@@ -57,9 +61,10 @@ class StringExpressionsITCase(mode: TestExecutionMode) extends MultipleProgramsT
@Test(expected = classOf[CodeGenException])
def testNonWorkingSubstring1(): Unit = {
-
val env = ExecutionEnvironment.getExecutionEnvironment
- val t = env.fromElements(("AAAA", 2.0), ("BBBB", 1.0)).as('a, 'b)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
+ val t = env.fromElements(("AAAA", 2.0), ("BBBB", 1.0)).toTable(tEnv, 'a, 'b)
// must fail, second argument of substring must be Integer not Double.
.select('a.substring(0, 'b))
@@ -68,9 +73,10 @@ class StringExpressionsITCase(mode: TestExecutionMode) extends MultipleProgramsT
@Test(expected = classOf[CodeGenException])
def testNonWorkingSubstring2(): Unit = {
-
val env = ExecutionEnvironment.getExecutionEnvironment
- val t = env.fromElements(("AAAA", "c"), ("BBBB", "d")).as('a, 'b)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
+ val t = env.fromElements(("AAAA", "c"), ("BBBB", "d")).toTable(tEnv, 'a, 'b)
// must fail, first argument of substring must be Integer not String.
.select('a.substring('b, 15))
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/TableEnvironmentITCase.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/TableEnvironmentITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/TableEnvironmentITCase.scala
new file mode 100644
index 0000000..bd1ce46
--- /dev/null
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/TableEnvironmentITCase.scala
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.scala.table.test
+
+import org.apache.flink.api.scala._
+import org.apache.flink.api.scala.table._
+import org.apache.flink.api.scala.util.CollectionDataSets
+import org.apache.flink.api.table.{TableEnvironment, TableException, Row}
+import org.apache.flink.api.table.test.utils.TableProgramsTestBase
+import org.apache.flink.api.table.test.utils.TableProgramsTestBase.TableConfigMode
+import org.apache.flink.test.util.MultipleProgramsTestBase.TestExecutionMode
+import org.apache.flink.test.util.TestBaseUtils
+import org.junit._
+import org.junit.runner.RunWith
+import org.junit.runners.Parameterized
+
+import scala.collection.JavaConverters._
+
+@RunWith(classOf[Parameterized])
+class TableEnvironmentITCase(
+ mode: TestExecutionMode,
+ configMode: TableConfigMode)
+ extends TableProgramsTestBase(mode, configMode) {
+
+ @Test
+ def testSimpleRegister(): Unit = {
+
+ val tableName = "MyTable"
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
+
+ val ds = CollectionDataSets.get3TupleDataSet(env)
+ tEnv.registerDataSet(tableName, ds)
+ val t = tEnv.scan(tableName).select('_1, '_2, '_3)
+
+ val expected = "1,1,Hi\n" + "2,2,Hello\n" + "3,2,Hello world\n" +
+ "4,3,Hello world, how are you?\n" + "5,3,I am fine.\n" + "6,3,Luke Skywalker\n" +
+ "7,4,Comment#1\n" + "8,4,Comment#2\n" + "9,4,Comment#3\n" + "10,4,Comment#4\n" +
+ "11,5,Comment#5\n" + "12,5,Comment#6\n" + "13,5,Comment#7\n" + "14,5,Comment#8\n" +
+ "15,5,Comment#9\n" + "16,6,Comment#10\n" + "17,6,Comment#11\n" + "18,6,Comment#12\n" +
+ "19,6,Comment#13\n" + "20,6,Comment#14\n" + "21,6,Comment#15\n"
+ val results = t.toDataSet[Row].collect()
+ TestBaseUtils.compareResultAsText(results.asJava, expected)
+ }
+
+ @Test
+ def testRegisterWithFields(): Unit = {
+
+ val tableName = "MyTable"
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
+
+ val ds = CollectionDataSets.get3TupleDataSet(env)
+ tEnv.registerDataSet(tableName, ds, 'a, 'b, 'c)
+ val t = tEnv.scan(tableName).select('a, 'b)
+
+ val expected = "1,1\n" + "2,2\n" + "3,2\n" + "4,3\n" + "5,3\n" + "6,3\n" +
+ "7,4\n" + "8,4\n" + "9,4\n" + "10,4\n" + "11,5\n" + "12,5\n" + "13,5\n" + "14,5\n" +
+ "15,5\n" + "16,6\n" + "17,6\n" + "18,6\n" + "19,6\n" + "20,6\n" + "21,6\n"
+ val results = t.toDataSet[Row].collect()
+ TestBaseUtils.compareResultAsText(results.asJava, expected)
+ }
+
+ @Test(expected = classOf[TableException])
+ def testRegisterExistingDataSet(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
+
+ val ds1 = CollectionDataSets.get3TupleDataSet(env)
+ tEnv.registerDataSet("MyTable", ds1)
+ val ds2 = CollectionDataSets.get5TupleDataSet(env)
+ // Must fail. Name is already in use.
+ tEnv.registerDataSet("MyTable", ds2)
+ }
+
+ @Test(expected = classOf[TableException])
+ def testScanUnregisteredTable(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
+ // Must fail. No table registered under that name.
+ tEnv.scan("someTable")
+ }
+
+ @Test
+ def testTableRegister(): Unit = {
+
+ val tableName = "MyTable"
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
+
+ val t = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv, 'a, 'b, 'c)
+ tEnv.registerTable(tableName, t)
+
+ val regT = tEnv.scan(tableName).select('a, 'b).filter('a > 8)
+
+ val expected = "9,4\n" + "10,4\n" +
+ "11,5\n" + "12,5\n" + "13,5\n" + "14,5\n" +
+ "15,5\n" + "16,6\n" + "17,6\n" + "18,6\n" +
+ "19,6\n" + "20,6\n" + "21,6\n"
+
+ val results = regT.toDataSet[Row].collect()
+ TestBaseUtils.compareResultAsText(results.asJava, expected)
+ }
+
+ @Test(expected = classOf[TableException])
+ def testRegisterExistingTable(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
+
+ val t1 = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv)
+ tEnv.registerTable("MyTable", t1)
+ val t2 = CollectionDataSets.get5TupleDataSet(env).toTable(tEnv)
+ // Must fail. Name is already in use.
+ tEnv.registerDataSet("MyTable", t2)
+ }
+
+ @Test(expected = classOf[TableException])
+ def testRegisterTableFromOtherEnv(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val tEnv1 = TableEnvironment.getTableEnvironment(env, config)
+ val tEnv2 = TableEnvironment.getTableEnvironment(env, config)
+
+ val t1 = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv1)
+ // Must fail. Table is bound to different TableEnvironment.
+ tEnv2.registerTable("MyTable", t1)
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/ToTableITCase.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/ToTableITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/ToTableITCase.scala
new file mode 100644
index 0000000..f162846
--- /dev/null
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/ToTableITCase.scala
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.scala.table.test
+
+import org.apache.flink.api.scala._
+import org.apache.flink.api.scala.table._
+import org.apache.flink.api.scala.util.CollectionDataSets
+import org.apache.flink.api.table.{TableEnvironment, Row}
+import org.apache.flink.api.table.test.utils.TableProgramsTestBase
+import TableProgramsTestBase.TableConfigMode
+import org.apache.flink.api.table.test.utils.TableProgramsTestBase
+import org.apache.flink.test.util.MultipleProgramsTestBase.TestExecutionMode
+import org.apache.flink.test.util.TestBaseUtils
+import org.junit._
+import org.junit.runner.RunWith
+import org.junit.runners.Parameterized
+
+import scala.collection.JavaConverters._
+
+@RunWith(classOf[Parameterized])
+class ToTableITCase(
+ mode: TestExecutionMode,
+ configMode: TableConfigMode)
+ extends TableProgramsTestBase(mode, configMode) {
+
+ @Test
+ def testToTable(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
+
+ val t = CollectionDataSets.get3TupleDataSet(env)
+ .toTable(tEnv, 'a, 'b, 'c)
+ .select('a, 'b, 'c)
+
+ val expected = "1,1,Hi\n" + "2,2,Hello\n" + "3,2,Hello world\n" +
+ "4,3,Hello world, how are you?\n" + "5,3,I am fine.\n" + "6,3,Luke Skywalker\n" +
+ "7,4,Comment#1\n" + "8,4,Comment#2\n" + "9,4,Comment#3\n" + "10,4,Comment#4\n" +
+ "11,5,Comment#5\n" + "12,5,Comment#6\n" + "13,5,Comment#7\n" + "14,5,Comment#8\n" +
+ "15,5,Comment#9\n" + "16,6,Comment#10\n" + "17,6,Comment#11\n" + "18,6,Comment#12\n" +
+ "19,6,Comment#13\n" + "20,6,Comment#14\n" + "21,6,Comment#15\n"
+ val results = t.toDataSet[Row].collect()
+ TestBaseUtils.compareResultAsText(results.asJava, expected)
+ }
+
+ @Test
+ def testToTableFromCaseClass(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
+
+ val data = List(
+ SomeCaseClass("Peter", 28, 4000.00, "Sales"),
+ SomeCaseClass("Anna", 56, 10000.00, "Engineering"),
+ SomeCaseClass("Lucy", 42, 6000.00, "HR"))
+
+ val t = env.fromCollection(data)
+ .toTable(tEnv, 'a, 'b, 'c, 'd)
+ .select('a, 'b, 'c, 'd)
+
+ val expected: String =
+ "Peter,28,4000.0,Sales\n" +
+ "Anna,56,10000.0,Engineering\n" +
+ "Lucy,42,6000.0,HR\n"
+ val results = t.toDataSet[Row].collect()
+ TestBaseUtils.compareResultAsText(results.asJava, expected)
+ }
+
+ @Test
+ def testToTableFromAndToCaseClass(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
+
+ val data = List(
+ SomeCaseClass("Peter", 28, 4000.00, "Sales"),
+ SomeCaseClass("Anna", 56, 10000.00, "Engineering"),
+ SomeCaseClass("Lucy", 42, 6000.00, "HR"))
+
+ val t = env.fromCollection(data)
+ .toTable(tEnv, 'a, 'b, 'c, 'd)
+ .select('a, 'b, 'c, 'd)
+
+ val expected: String =
+ "SomeCaseClass(Peter,28,4000.0,Sales)\n" +
+ "SomeCaseClass(Anna,56,10000.0,Engineering)\n" +
+ "SomeCaseClass(Lucy,42,6000.0,HR)\n"
+ val results = t.toDataSet[SomeCaseClass].collect()
+ TestBaseUtils.compareResultAsText(results.asJava, expected)
+ }
+
+ @Test(expected = classOf[IllegalArgumentException])
+ def testToTableWithToFewFields(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
+
+ CollectionDataSets.get3TupleDataSet(env)
+ // Must fail. Number of fields does not match.
+ .toTable(tEnv, 'a, 'b)
+ }
+
+ @Test(expected = classOf[IllegalArgumentException])
+ def testToTableWithToManyFields(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
+
+ CollectionDataSets.get3TupleDataSet(env)
+ // Must fail. Number of fields does not match.
+ .toTable(tEnv, 'a, 'b, 'c, 'd)
+ }
+
+ @Test(expected = classOf[IllegalArgumentException])
+ def testToTableWithAmbiguousFields(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
+
+ CollectionDataSets.get3TupleDataSet(env)
+ // Must fail. Field names not unique.
+ .toTable(tEnv, 'a, 'b, 'b)
+ }
+
+ @Test(expected = classOf[IllegalArgumentException])
+ def testToTableWithNonFieldReference1(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
+
+ // Must fail. as() can only have field references
+ CollectionDataSets.get3TupleDataSet(env)
+ .toTable(tEnv, 'a + 1, 'b, 'c)
+ }
+
+ @Test(expected = classOf[IllegalArgumentException])
+ def testToTableWithNonFieldReference2(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
+
+ // Must fail. as() can only have field references
+ CollectionDataSets.get3TupleDataSet(env)
+ .toTable(tEnv, 'a as 'foo, 'b, 'c)
+ }
+
+}
+
+case class SomeCaseClass(name: String, age: Int, salary: Double, department: String) {
+ def this() { this("", 0, 0.0, "") }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/UnionITCase.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/UnionITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/UnionITCase.scala
index a155935..0448386 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/UnionITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/UnionITCase.scala
@@ -21,7 +21,7 @@ package org.apache.flink.api.scala.table.test
import org.apache.flink.api.scala._
import org.apache.flink.api.scala.table._
import org.apache.flink.api.scala.util.CollectionDataSets
-import org.apache.flink.api.table.Row
+import org.apache.flink.api.table.{TableException, TableEnvironment, Row}
import org.apache.flink.test.util.MultipleProgramsTestBase.TestExecutionMode
import org.apache.flink.test.util.TestBaseUtils
import org.junit._
@@ -40,8 +40,10 @@ class UnionITCase(
@Test
def testUnion(): Unit = {
val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = CollectionDataSets.getSmall3TupleDataSet(env).as('a, 'b, 'c)
- val ds2 = CollectionDataSets.getSmall3TupleDataSet(env).as('a, 'b, 'c)
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
+
+ val ds1 = CollectionDataSets.getSmall3TupleDataSet(env).toTable(tEnv, 'a, 'b, 'c)
+ val ds2 = CollectionDataSets.getSmall3TupleDataSet(env).toTable(tEnv, 'a, 'b, 'c)
val unionDs = ds1.unionAll(ds2).select('c)
@@ -53,9 +55,11 @@ class UnionITCase(
@Test
def testTernaryUnion(): Unit = {
val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = CollectionDataSets.getSmall3TupleDataSet(env).as('a, 'b, 'c)
- val ds2 = CollectionDataSets.getSmall3TupleDataSet(env).as('a, 'b, 'c)
- val ds3 = CollectionDataSets.getSmall3TupleDataSet(env).as('a, 'b, 'c)
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
+
+ val ds1 = CollectionDataSets.getSmall3TupleDataSet(env).toTable(tEnv, 'a, 'b, 'c)
+ val ds2 = CollectionDataSets.getSmall3TupleDataSet(env).toTable(tEnv, 'a, 'b, 'c)
+ val ds3 = CollectionDataSets.getSmall3TupleDataSet(env).toTable(tEnv, 'a, 'b, 'c)
val unionDs = ds1.unionAll(ds2).unionAll(ds3).select('c)
@@ -69,8 +73,14 @@ class UnionITCase(
@Test
def testUnionWithFilter(): Unit = {
val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = CollectionDataSets.getSmall3TupleDataSet(env).as('a, 'b, 'c)
- val ds2 = CollectionDataSets.get5TupleDataSet(env).as('a, 'b, 'd, 'c, 'e)
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
+
+ if (tEnv.getConfig.getEfficientTypeUsage) {
+ return
+ }
+
+ val ds1 = CollectionDataSets.getSmall3TupleDataSet(env).toTable(tEnv, 'a, 'b, 'c)
+ val ds2 = CollectionDataSets.get5TupleDataSet(env).toTable(tEnv, 'a, 'b, 'd, 'c, 'e)
val joinDs = ds1.unionAll(ds2.select('a, 'b, 'c)).filter('b < 2).select('c)
@@ -82,8 +92,10 @@ class UnionITCase(
@Test(expected = classOf[IllegalArgumentException])
def testUnionDifferentFieldNames(): Unit = {
val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = CollectionDataSets.getSmall3TupleDataSet(env).as('a, 'b, 'c)
- val ds2 = CollectionDataSets.get5TupleDataSet(env).as('a, 'b, 'd, 'c, 'e)
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
+
+ val ds1 = CollectionDataSets.getSmall3TupleDataSet(env).toTable(tEnv, 'a, 'b, 'c)
+ val ds2 = CollectionDataSets.get5TupleDataSet(env).toTable(tEnv, 'a, 'b, 'd, 'c, 'e)
// must fail. Union inputs have different field names.
ds1.unionAll(ds2)
@@ -92,8 +104,11 @@ class UnionITCase(
@Test(expected = classOf[IllegalArgumentException])
def testUnionDifferentFieldTypes(): Unit = {
val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = CollectionDataSets.getSmall3TupleDataSet(env).as('a, 'b, 'c)
- val ds2 = CollectionDataSets.get5TupleDataSet(env).as('a, 'b, 'c, 'd, 'e).select('a, 'b, 'c)
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
+
+ val ds1 = CollectionDataSets.getSmall3TupleDataSet(env).toTable(tEnv, 'a, 'b, 'c)
+ val ds2 = CollectionDataSets.get5TupleDataSet(env).toTable(tEnv, 'a, 'b, 'c, 'd, 'e)
+ .select('a, 'b, 'c)
// must fail. Union inputs have different field types.
ds1.unionAll(ds2)
@@ -102,8 +117,14 @@ class UnionITCase(
@Test
def testUnionWithAggregation(): Unit = {
val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = CollectionDataSets.getSmall3TupleDataSet(env).as('a, 'b, 'c)
- val ds2 = CollectionDataSets.get5TupleDataSet(env).as('a, 'b, 'd, 'c, 'e)
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
+
+ if (tEnv.getConfig.getEfficientTypeUsage) {
+ return
+ }
+
+ val ds1 = CollectionDataSets.getSmall3TupleDataSet(env).toTable(tEnv, 'a, 'b, 'c)
+ val ds2 = CollectionDataSets.get5TupleDataSet(env).toTable(tEnv, 'a, 'b, 'd, 'c, 'e)
val unionDs = ds1.unionAll(ds2.select('a, 'b, 'c)).select('c.count)
@@ -115,9 +136,15 @@ class UnionITCase(
@Test
def testUnionWithJoin(): Unit = {
val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = CollectionDataSets.getSmall3TupleDataSet(env).as('a, 'b, 'c)
- val ds2 = CollectionDataSets.get5TupleDataSet(env).as('a, 'b, 'd, 'c, 'e)
- val ds3 = CollectionDataSets.getSmall5TupleDataSet(env).as('a2, 'b2, 'd2, 'c2, 'e2)
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
+
+ if (tEnv.getConfig.getEfficientTypeUsage) {
+ return
+ }
+
+ val ds1 = CollectionDataSets.getSmall3TupleDataSet(env).toTable(tEnv, 'a, 'b, 'c)
+ val ds2 = CollectionDataSets.get5TupleDataSet(env).toTable(tEnv,'a, 'b, 'd, 'c, 'e)
+ val ds3 = CollectionDataSets.getSmall5TupleDataSet(env).toTable(tEnv, 'a2, 'b2, 'd2, 'c2, 'e2)
val joinDs = ds1.unionAll(ds2.select('a, 'b, 'c))
.join(ds3.select('a2, 'b2, 'c2))
@@ -130,4 +157,17 @@ class UnionITCase(
"Hallo Welt,Hallo Welt wie\n" + "Hallo Welt wie,Hallo Welt wie\n"
TestBaseUtils.compareResultAsText(results.asJava, expected)
}
+
+ @Test(expected = classOf[TableException])
+ def testUnionTablesFromDifferentEnvs(): Unit = {
+ val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
+ val tEnv1 = TableEnvironment.getTableEnvironment(env, config)
+ val tEnv2 = TableEnvironment.getTableEnvironment(env, config)
+
+ val ds1 = CollectionDataSets.getSmall3TupleDataSet(env).toTable(tEnv1, 'a, 'b, 'c)
+ val ds2 = CollectionDataSets.getSmall3TupleDataSet(env).toTable(tEnv2, 'a, 'b, 'c)
+
+ // Must fail. Tables are bound to different TableEnvironments.
+ ds1.unionAll(ds2).select('c)
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/table/test/utils/ExpressionEvaluator.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/table/test/utils/ExpressionEvaluator.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/table/test/utils/ExpressionEvaluator.scala
index 48dea56..4e1ae02 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/table/test/utils/ExpressionEvaluator.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/table/test/utils/ExpressionEvaluator.scala
@@ -25,12 +25,11 @@ import org.apache.calcite.tools.{Frameworks, RelBuilder}
import org.apache.flink.api.common.functions.{Function, MapFunction}
import org.apache.flink.api.common.typeinfo.BasicTypeInfo._
import org.apache.flink.api.common.typeinfo.TypeInformation
-import org.apache.flink.api.java.DataSet
-import org.apache.flink.api.table.TableConfig
+import org.apache.flink.api.scala.{DataSet, ExecutionEnvironment}
+import org.apache.flink.api.java.{DataSet => JDataSet}
+import org.apache.flink.api.table.{TableEnvironment, TableConfig}
import org.apache.flink.api.table.codegen.{CodeGenerator, GeneratedFunction}
import org.apache.flink.api.table.expressions.Expression
-import org.apache.flink.api.table.plan.TranslationContext
-import org.apache.flink.api.table.plan.schema.DataSetTable
import org.apache.flink.api.table.runtime.FunctionCompiler
import org.mockito.Mockito._
@@ -45,20 +44,26 @@ object ExpressionEvaluator {
compile(getClass.getClassLoader, genFunc.name, genFunc.code)
}
- private def prepareTable(typeInfo: TypeInformation[Any]): (String, RelBuilder) = {
+ private def prepareTable(
+ typeInfo: TypeInformation[Any]): (String, RelBuilder, TableEnvironment) = {
+
// create DataSetTable
val dataSetMock = mock(classOf[DataSet[Any]])
- when(dataSetMock.getType).thenReturn(typeInfo)
- val tableName = TranslationContext.registerDataSetTable(new DataSetTable[Any](
- dataSetMock,
- (0 until typeInfo.getArity).toArray,
- (0 until typeInfo.getArity).map("f" + _).toArray))
+ val jDataSetMock = mock(classOf[JDataSet[Any]])
+ when(dataSetMock.javaSet).thenReturn(jDataSetMock)
+ when(jDataSetMock.getType).thenReturn(typeInfo)
+
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
+ val tableName = "myTable"
+ tEnv.registerDataSet(tableName, dataSetMock)
// prepare RelBuilder
- val relBuilder = TranslationContext.getRelBuilder
+ val relBuilder = tEnv.getRelBuilder
relBuilder.scan(tableName)
- (tableName, relBuilder)
+ (tableName, relBuilder, tEnv)
}
def evaluate(data: Any, typeInfo: TypeInformation[Any], sqlExpr: String): String = {
@@ -66,7 +71,7 @@ object ExpressionEvaluator {
val table = prepareTable(typeInfo)
// create RelNode from SQL expression
- val planner = Frameworks.getPlanner(TranslationContext.getFrameworkConfig)
+ val planner = Frameworks.getPlanner(table._3.getFrameworkConfig)
val parsed = planner.parse("SELECT " + sqlExpr + " FROM " + table._1)
val validated = planner.validate(parsed)
val converted = planner.rel(validated)
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/table/test/utils/TableProgramsTestBase.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/table/test/utils/TableProgramsTestBase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/table/test/utils/TableProgramsTestBase.scala
index a971136..723646b 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/table/test/utils/TableProgramsTestBase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/table/test/utils/TableProgramsTestBase.scala
@@ -20,11 +20,11 @@ package org.apache.flink.api.table.test.utils
import java.util
-import org.apache.flink.api.java.table.{TableEnvironment => JavaTableEnv}
+import org.apache.flink.api.java.table.{BatchTableEnvironment => JavaTableEnv}
import org.apache.flink.api.java.{ExecutionEnvironment => JavaEnv}
-import org.apache.flink.api.scala.table.{TableEnvironment => ScalaTableEnv}
+import org.apache.flink.api.scala.table.{BatchTableEnvironment => ScalaTableEnv}
import org.apache.flink.api.scala.{ExecutionEnvironment => ScalaEnv}
-import org.apache.flink.api.table.TableConfig
+import org.apache.flink.api.table.{TableEnvironment, TableConfig}
import org.apache.flink.api.table.test.utils.TableProgramsTestBase.TableConfigMode
import org.apache.flink.api.table.test.utils.TableProgramsTestBase.TableConfigMode.{EFFICIENT, NULL}
import org.apache.flink.test.util.MultipleProgramsTestBase
@@ -38,34 +38,16 @@ class TableProgramsTestBase(
tableConfigMode: TableConfigMode)
extends MultipleProgramsTestBase(mode) {
- def getJavaTableEnvironment: JavaTableEnv = {
- val env = JavaEnv.getExecutionEnvironment // TODO pass it to tableEnv
- val tableEnv = new JavaTableEnv
- configure(tableEnv.getConfig)
- tableEnv
- }
-
- def getScalaTableEnvironment: ScalaTableEnv = {
- val env = ScalaEnv.getExecutionEnvironment // TODO pass it to tableEnv
- val tableEnv = new ScalaTableEnv
- configure(tableEnv.getConfig)
- tableEnv
- }
-
- def getConfig: TableConfig = {
- val config = new TableConfig()
- configure(config)
- config
- }
-
- def configure(config: TableConfig): Unit = {
+ def config: TableConfig = {
+ val conf = new TableConfig
tableConfigMode match {
case NULL =>
- config.setNullCheck(true)
+ conf.setNullCheck(true)
case EFFICIENT =>
- config.setEfficientTypeUsage(true)
+ conf.setEfficientTypeUsage(true)
case _ => // keep default
}
+ conf
}
}
[11/12] flink git commit: [FLINK-3762] [core] Enable Kryo reference
tracking
Posted by fh...@apache.org.
[FLINK-3762] [core] Enable Kryo reference tracking
This closes #1891
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/dc78a747
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/dc78a747
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/dc78a747
Branch: refs/heads/master
Commit: dc78a7470a5da086a08140b200a20d840460ef79
Parents: f3d3a44
Author: Andrew Palumbo <ap...@apache.org>
Authored: Thu Apr 14 14:25:46 2016 -0400
Committer: Fabian Hueske <fh...@apache.org>
Committed: Fri Apr 15 19:38:15 2016 +0200
----------------------------------------------------------------------
.../flink/api/java/typeutils/runtime/kryo/KryoSerializer.java | 5 ++---
.../typeutils/runtime/kryo/KryoGenericTypeSerializerTest.java | 4 ++--
2 files changed, 4 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/dc78a747/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializer.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializer.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializer.java
index d5c2f67..e74e251 100644
--- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializer.java
@@ -327,9 +327,8 @@ public class KryoSerializer<T> extends TypeSerializer<T> {
if (this.kryo == null) {
this.kryo = getKryoInstance();
- // disable reference tracking. reference tracking is costly, usually unnecessary, and
- // inconsistent with Flink's own serialization (which does not do reference tracking)
- kryo.setReferences(false);
+ // Enable reference tracking.
+ kryo.setReferences(true);
// Throwable and all subclasses should be serialized via java serialization
kryo.addDefaultSerializer(Throwable.class, new JavaSerializer());
http://git-wip-us.apache.org/repos/asf/flink/blob/dc78a747/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoGenericTypeSerializerTest.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoGenericTypeSerializerTest.java b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoGenericTypeSerializerTest.java
index 8ff0b1b..12090f8 100644
--- a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoGenericTypeSerializerTest.java
+++ b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoGenericTypeSerializerTest.java
@@ -160,9 +160,9 @@ public class KryoGenericTypeSerializerTest extends AbstractGenericTypeSerializer
}
@Test
- public void validateReferenceMappingDisabled() {
+ public void validateReferenceMappingEnabled() {
KryoSerializer<String> serializer = new KryoSerializer<>(String.class, new ExecutionConfig());
Kryo kryo = serializer.getKryo();
- assertFalse(kryo.getReferences());
+ assertTrue(kryo.getReferences());
}
}
\ No newline at end of file
[05/12] flink git commit: [FLINK-3732] [core] Fix potential null
deference in ExecutionConfig#equals()
Posted by fh...@apache.org.
[FLINK-3732] [core] Fix potential null deference in ExecutionConfig#equals()
This closes #1871
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/f3d3a449
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/f3d3a449
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/f3d3a449
Branch: refs/heads/master
Commit: f3d3a4493ae786d421176396ee68f01a0e6dbb64
Parents: 722155a
Author: Tianji Li <ti...@alcatel-lucent.com>
Authored: Mon Apr 11 13:56:34 2016 -0400
Committer: Fabian Hueske <fh...@apache.org>
Committed: Fri Apr 15 19:38:15 2016 +0200
----------------------------------------------------------------------
.../src/main/java/org/apache/flink/api/common/ExecutionConfig.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/f3d3a449/flink-core/src/main/java/org/apache/flink/api/common/ExecutionConfig.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/ExecutionConfig.java b/flink-core/src/main/java/org/apache/flink/api/common/ExecutionConfig.java
index c02bcad..c27ee74 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/ExecutionConfig.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/ExecutionConfig.java
@@ -778,7 +778,7 @@ public class ExecutionConfig implements Serializable {
useClosureCleaner == other.useClosureCleaner &&
parallelism == other.parallelism &&
((restartStrategyConfiguration == null && other.restartStrategyConfiguration == null) ||
- restartStrategyConfiguration.equals(other.restartStrategyConfiguration)) &&
+ (null != restartStrategyConfiguration && restartStrategyConfiguration.equals(other.restartStrategyConfiguration))) &&
forceKryo == other.forceKryo &&
objectReuse == other.objectReuse &&
autoTypeRegistrationEnabled == other.autoTypeRegistrationEnabled &&
[02/12] flink git commit: [FLINK-3700] [core] Remove Guava dependency
from flink-core
Posted by fh...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-core/src/main/java/org/apache/flink/types/StringValue.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/types/StringValue.java b/flink-core/src/main/java/org/apache/flink/types/StringValue.java
index e20083e..92f364e 100644
--- a/flink-core/src/main/java/org/apache/flink/types/StringValue.java
+++ b/flink-core/src/main/java/org/apache/flink/types/StringValue.java
@@ -28,7 +28,7 @@ import org.apache.flink.core.memory.DataInputView;
import org.apache.flink.core.memory.DataOutputView;
import org.apache.flink.core.memory.MemorySegment;
-import com.google.common.base.Preconditions;
+import static org.apache.flink.util.Preconditions.checkNotNull;
/**
* Mutable string data type that implements the Key interface.
@@ -147,7 +147,7 @@ public class StringValue implements NormalizableKey<StringValue>, CharSequence,
* @param value The new string value.
*/
public void setValue(CharSequence value) {
- Preconditions.checkNotNull(value);
+ checkNotNull(value);
setValue(value, 0, value.length());
}
@@ -158,7 +158,7 @@ public class StringValue implements NormalizableKey<StringValue>, CharSequence,
*/
@Override
public void setValue(StringValue value) {
- Preconditions.checkNotNull(value);
+ checkNotNull(value);
setValue(value.value, 0, value.len);
}
@@ -170,7 +170,7 @@ public class StringValue implements NormalizableKey<StringValue>, CharSequence,
* @param len The length of the substring.
*/
public void setValue(StringValue value, int offset, int len) {
- Preconditions.checkNotNull(value);
+ checkNotNull(value);
setValue(value.value, offset, len);
}
@@ -182,7 +182,7 @@ public class StringValue implements NormalizableKey<StringValue>, CharSequence,
* @param len The length of the substring.
*/
public void setValue(CharSequence value, int offset, int len) {
- Preconditions.checkNotNull(value);
+ checkNotNull(value);
if (offset < 0 || len < 0 || offset > value.length() - len) {
throw new IndexOutOfBoundsException("offset: " + offset + " len: " + len + " value.len: " + len);
}
@@ -204,7 +204,7 @@ public class StringValue implements NormalizableKey<StringValue>, CharSequence,
* @param buffer The character buffer to read the characters from.
*/
public void setValue(CharBuffer buffer) {
- Preconditions.checkNotNull(buffer);
+ checkNotNull(buffer);
final int len = buffer.length();
ensureSize(len);
buffer.get(this.value, 0, len);
@@ -220,7 +220,7 @@ public class StringValue implements NormalizableKey<StringValue>, CharSequence,
* @param len The length of the substring.
*/
public void setValue(char[] chars, int offset, int len) {
- Preconditions.checkNotNull(chars);
+ checkNotNull(chars);
if (offset < 0 || len < 0 || offset > chars.length - len) {
throw new IndexOutOfBoundsException();
}
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-core/src/main/java/org/apache/flink/util/FileUtils.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/util/FileUtils.java b/flink-core/src/main/java/org/apache/flink/util/FileUtils.java
new file mode 100644
index 0000000..078599d
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/util/FileUtils.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.util;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.StandardOpenOption;
+
+/**
+ * This is a utility class to deal with temporary files.
+ */
+public final class FileUtils {
+
+ /**
+ * The alphabet to construct the random part of the filename from.
+ */
+ private static final char[] ALPHABET = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '0', 'a', 'b', 'c', 'd',
+ 'e', 'f' };
+
+ /**
+ * The length of the random part of the filename.
+ */
+ private static final int LENGTH = 12;
+
+
+
+ /**
+ * Constructs a random filename with the given prefix and
+ * a random part generated from hex characters.
+ *
+ * @param prefix
+ * the prefix to the filename to be constructed
+ * @return the generated random filename with the given prefix
+ */
+ public static String getRandomFilename(final String prefix) {
+
+ final StringBuilder stringBuilder = new StringBuilder(prefix);
+
+ for (int i = 0; i < LENGTH; i++) {
+ stringBuilder.append(ALPHABET[(int) Math.floor(Math.random() * (double) ALPHABET.length)]);
+ }
+
+ return stringBuilder.toString();
+ }
+
+ // ------------------------------------------------------------------------
+ // Simple reading and writing of files
+ // ------------------------------------------------------------------------
+
+ public static String readFile(File file, String charsetName) throws IOException {
+ byte[] bytes = Files.readAllBytes(file.toPath());
+ return new String(bytes, charsetName);
+ }
+
+ public static String readFileUtf8(File file) throws IOException {
+ return readFile(file, "UTF-8");
+ }
+
+ public static void writeFile(File file, String contents, String encoding) throws IOException {
+ byte[] bytes = contents.getBytes(encoding);
+ Files.write(file.toPath(), bytes, StandardOpenOption.WRITE);
+ }
+
+ public static void writeFileUtf8(File file, String contents) throws IOException {
+ writeFile(file, contents, "UTF-8");
+ }
+
+ // ------------------------------------------------------------------------
+
+ /**
+ * Private default constructor to avoid instantiation.
+ */
+ private FileUtils() {}
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-core/src/main/java/org/apache/flink/util/IOUtils.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/util/IOUtils.java b/flink-core/src/main/java/org/apache/flink/util/IOUtils.java
new file mode 100644
index 0000000..12d70ce
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/util/IOUtils.java
@@ -0,0 +1,223 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.util;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.PrintStream;
+import java.net.Socket;
+
+import org.slf4j.Logger;
+
+/**
+ * An utility class for I/O related functionality.
+ *
+ */
+public final class IOUtils {
+
+ /** The block size for byte operations in byte. */
+ private static final int BLOCKSIZE = 4096;
+
+ // ------------------------------------------------------------------------
+ // Byte copy operations
+ // ------------------------------------------------------------------------
+
+ /**
+ * Copies from one stream to another.
+ *
+ * @param in
+ * InputStream to read from
+ * @param out
+ * OutputStream to write to
+ * @param buffSize
+ * the size of the buffer
+ * @param close
+ * whether or not close the InputStream and OutputStream at the end. The streams are closed in the finally
+ * clause.
+ * @throws IOException
+ * thrown if an error occurred while writing to the output stream
+ */
+ public static void copyBytes(final InputStream in, final OutputStream out, final int buffSize, final boolean close)
+ throws IOException {
+
+ @SuppressWarnings("resource")
+ final PrintStream ps = out instanceof PrintStream ? (PrintStream) out : null;
+ final byte[] buf = new byte[buffSize];
+ try {
+ int bytesRead = in.read(buf);
+ while (bytesRead >= 0) {
+ out.write(buf, 0, bytesRead);
+ if ((ps != null) && ps.checkError()) {
+ throw new IOException("Unable to write to output stream.");
+ }
+ bytesRead = in.read(buf);
+ }
+ } finally {
+ if (close) {
+ out.close();
+ in.close();
+ }
+ }
+ }
+
+ /**
+ * Copies from one stream to another. <strong>closes the input and output
+ * streams at the end</strong>.
+ *
+ * @param in
+ * InputStream to read from
+ * @param out
+ * OutputStream to write to
+ * @throws IOException
+ * thrown if an I/O error occurs while copying
+ */
+ public static void copyBytes(final InputStream in, final OutputStream out) throws IOException {
+ copyBytes(in, out, BLOCKSIZE, true);
+ }
+
+ /**
+ * Copies from one stream to another.
+ *
+ * @param in
+ * InputStream to read from
+ * @param out
+ * OutputStream to write to
+ * @param close
+ * whether or not close the InputStream and OutputStream at the
+ * end. The streams are closed in the finally clause.
+ * @throws IOException
+ * thrown if an I/O error occurs while copying
+ */
+ public static void copyBytes(final InputStream in, final OutputStream out, final boolean close) throws IOException {
+ copyBytes(in, out, BLOCKSIZE, close);
+ }
+
+ // ------------------------------------------------------------------------
+ // Stream input skipping
+ // ------------------------------------------------------------------------
+
+ /**
+ * Reads len bytes in a loop.
+ *
+ * @param in
+ * The InputStream to read from
+ * @param buf
+ * The buffer to fill
+ * @param off
+ * offset from the buffer
+ * @param len
+ * the length of bytes to read
+ * @throws IOException
+ * if it could not read requested number of bytes for any reason (including EOF)
+ */
+ public static void readFully(final InputStream in, final byte[] buf, int off, final int len)
+ throws IOException {
+ int toRead = len;
+ while (toRead > 0) {
+ final int ret = in.read(buf, off, toRead);
+ if (ret < 0) {
+ throw new IOException("Premeture EOF from inputStream");
+ }
+ toRead -= ret;
+ off += ret;
+ }
+ }
+
+ /**
+ * Similar to readFully(). Skips bytes in a loop.
+ *
+ * @param in
+ * The InputStream to skip bytes from
+ * @param len
+ * number of bytes to skip
+ * @throws IOException
+ * if it could not skip requested number of bytes for any reason (including EOF)
+ */
+ public static void skipFully(final InputStream in, long len) throws IOException {
+ while (len > 0) {
+ final long ret = in.skip(len);
+ if (ret < 0) {
+ throw new IOException("Premeture EOF from inputStream");
+ }
+ len -= ret;
+ }
+ }
+
+ // ------------------------------------------------------------------------
+ // Silent I/O cleanup / closing
+ // ------------------------------------------------------------------------
+
+ /**
+ * Close the Closeable objects and <b>ignore</b> any {@link IOException} or
+ * null pointers. Must only be used for cleanup in exception handlers.
+ *
+ * @param log
+ * the log to record problems to at debug level. Can be <code>null</code>.
+ * @param closeables
+ * the objects to close
+ */
+ public static void cleanup(final Logger log, final java.io.Closeable... closeables) {
+ for (java.io.Closeable c : closeables) {
+ if (c != null) {
+ try {
+ c.close();
+ } catch (IOException e) {
+ if (log != null && log.isDebugEnabled()) {
+ log.debug("Exception in closing " + c, e);
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Closes the stream ignoring {@link IOException}. Must only be called in
+ * cleaning up from exception handlers.
+ *
+ * @param stream
+ * the stream to close
+ */
+ public static void closeStream(final java.io.Closeable stream) {
+ cleanup(null, stream);
+ }
+
+ /**
+ * Closes the socket ignoring {@link IOException}.
+ *
+ * @param sock
+ * the socket to close
+ */
+ public static void closeSocket(final Socket sock) {
+ // avoids try { close() } dance
+ if (sock != null) {
+ try {
+ sock.close();
+ } catch (IOException ignored) {
+ }
+ }
+ }
+
+ // ------------------------------------------------------------------------
+
+ /**
+ * Private constructor to prevent instantiation.
+ */
+ private IOUtils() {}
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-core/src/main/java/org/apache/flink/util/MathUtils.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/util/MathUtils.java b/flink-core/src/main/java/org/apache/flink/util/MathUtils.java
new file mode 100644
index 0000000..2bdddc8
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/util/MathUtils.java
@@ -0,0 +1,175 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.util;
+
+/**
+ * Collection of simple mathematical routines.
+ */
+public final class MathUtils {
+
+ /**
+ * Computes the logarithm of the given value to the base of 2, rounded down. It corresponds to the
+ * position of the highest non-zero bit. The position is counted, starting with 0 from the least
+ * significant bit to the most significant bit. For example, <code>log2floor(16) = 4</code>, and
+ * <code>log2floor(10) = 3</code>.
+ *
+ * @param value The value to compute the logarithm for.
+ * @return The logarithm (rounded down) to the base of 2.
+ * @throws ArithmeticException Thrown, if the given value is zero.
+ */
+ public static int log2floor(int value) throws ArithmeticException {
+ if (value == 0) {
+ throw new ArithmeticException("Logarithm of zero is undefined.");
+ }
+
+ int log = 0;
+ while ((value = value >>> 1) != 0) {
+ log++;
+ }
+
+ return log;
+ }
+
+ /**
+ * Computes the logarithm of the given value to the base of 2. This method throws an error,
+ * if the given argument is not a power of 2.
+ *
+ * @param value The value to compute the logarithm for.
+ * @return The logarithm to the base of 2.
+ * @throws ArithmeticException Thrown, if the given value is zero.
+ * @throws IllegalArgumentException Thrown, if the given value is not a power of two.
+ */
+ public static int log2strict(int value) throws ArithmeticException, IllegalArgumentException {
+ if (value == 0) {
+ throw new ArithmeticException("Logarithm of zero is undefined.");
+ }
+ if ((value & (value - 1)) != 0) {
+ throw new IllegalArgumentException("The given value " + value + " is not a power of two.");
+ }
+
+ int log = 0;
+ while ((value = value >>> 1) != 0) {
+ log++;
+ }
+
+ return log;
+ }
+
+ /**
+ * Decrements the given number down to the closest power of two. If the argument is a
+ * power of two, it remains unchanged.
+ *
+ * @param value The value to round down.
+ * @return The closest value that is a power of to and less or equal than the given value.
+ */
+ public static int roundDownToPowerOf2(int value) {
+ return Integer.highestOneBit(value);
+ }
+
+ /**
+ * Casts the given value to a 32 bit integer, if it can be safely done. If the cast would change the numeric
+ * value, this method raises an exception.
+ * <p>
+ * This method is a protection in places where one expects to be able to safely case, but where unexpected
+ * situations could make the cast unsafe and would cause hidden problems that are hard to track down.
+ *
+ * @param value The value to be cast to an integer.
+ * @return The given value as an integer.
+ */
+ public static int checkedDownCast(long value) {
+ if (value > Integer.MAX_VALUE) {
+ throw new IllegalArgumentException("Cannot downcast long value " + value + " to integer.");
+ }
+ return (int) value;
+ }
+
+ /**
+ * Checks whether the given value is a power of two.
+ *
+ * @param value The value to check.
+ * @return True, if the value is a power of two, false otherwise.
+ */
+ public static boolean isPowerOf2(long value) {
+ return (value & (value - 1)) == 0;
+ }
+
+ /**
+ * This function hashes an integer value. It is adapted from Bob Jenkins' website
+ * <a href="http://www.burtleburtle.net/bob/hash/integer.html">http://www.burtleburtle.net/bob/hash/integer.html</a>.
+ * The hash function has the <i>full avalanche</i> property, meaning that every bit of the value to be hashed
+ * affects every bit of the hash value.
+ *
+ * It is crucial to use different hash functions to partition data across machines and the internal partitioning of
+ * data structures. This hash function is intended for partitioning internally in data structures.
+ *
+ * @param code The integer to be hashed.
+ * @return The non-negative hash code for the integer.
+ */
+ public static int jenkinsHash(int code) {
+ code = (code + 0x7ed55d16) + (code << 12);
+ code = (code ^ 0xc761c23c) ^ (code >>> 19);
+ code = (code + 0x165667b1) + (code << 5);
+ code = (code + 0xd3a2646c) ^ (code << 9);
+ code = (code + 0xfd7046c5) + (code << 3);
+ code = (code ^ 0xb55a4f09) ^ (code >>> 16);
+ return code >= 0 ? code : -(code + 1);
+ }
+
+ /**
+ * This function hashes an integer value.
+ *
+ * It is crucial to use different hash functions to partition data across machines and the internal partitioning of
+ * data structures. This hash function is intended for partitioning across machines.
+ *
+ * @param code The integer to be hashed.
+ * @return The non-negative hash code for the integer.
+ */
+ public static int murmurHash(int code) {
+ code *= 0xcc9e2d51;
+ code = Integer.rotateLeft(code, 15);
+ code *= 0x1b873593;
+
+ code = Integer.rotateLeft(code, 13);
+ code = code * 5 + 0xe6546b64;
+
+ code ^= 4;
+ code ^= code >>> 16;
+ code *= 0x85ebca6b;
+ code ^= code >>> 13;
+ code *= 0xc2b2ae35;
+ code ^= code >>> 16;
+
+ if (code >= 0) {
+ return code;
+ }
+ else if (code != Integer.MIN_VALUE) {
+ return -code;
+ }
+ else {
+ return 0;
+ }
+ }
+
+ // ============================================================================================
+
+ /**
+ * Prevent Instantiation through private constructor.
+ */
+ private MathUtils() {}
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-core/src/main/java/org/apache/flink/util/NetUtils.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/util/NetUtils.java b/flink-core/src/main/java/org/apache/flink/util/NetUtils.java
index d9c4d3c..6f63eb4 100644
--- a/flink-core/src/main/java/org/apache/flink/util/NetUtils.java
+++ b/flink-core/src/main/java/org/apache/flink/util/NetUtils.java
@@ -18,9 +18,8 @@
package org.apache.flink.util;
-import com.google.common.collect.Iterators;
-import com.google.common.net.InetAddresses;
import org.apache.flink.annotation.Internal;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -33,9 +32,9 @@ import java.net.MalformedURLException;
import java.net.ServerSocket;
import java.net.URL;
import java.net.UnknownHostException;
-import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
import java.util.Iterator;
-import java.util.List;
@Internal
public class NetUtils {
@@ -116,8 +115,6 @@ public class NetUtils {
/**
* Encodes an IP address properly as a URL string. This method makes sure that IPv6 addresses
* have the proper formatting to be included in URLs.
- * <p>
- * This method internally uses Guava's functionality to properly encode IPv6 addresses.
*
* @param address The IP address to encode.
* @return The proper URL string encoded IP address.
@@ -130,7 +127,7 @@ public class NetUtils {
return address.getHostAddress();
}
else if (address instanceof Inet6Address) {
- return '[' + InetAddresses.toAddrString(address) + ']';
+ return getIPv6UrlRepresentation((Inet6Address) address);
}
else {
throw new IllegalArgumentException("Unrecognized type of InetAddress: " + address);
@@ -178,6 +175,70 @@ public class NetUtils {
}
/**
+ * Creates a compressed URL style representation of an Inet6Address.
+ *
+ * <p>This method copies and adopts code from Google's Guava library.
+ * We re-implement this here in order to reduce dependency on Guava.
+ * The Guava library has frequently caused dependency conflicts in the past.
+ */
+ private static String getIPv6UrlRepresentation(Inet6Address address) {
+ // first, convert bytes to 16 bit chunks
+ byte[] addressBytes = address.getAddress();
+ int[] hextets = new int[8];
+ for (int i = 0; i < hextets.length; i++) {
+ hextets[i] = (addressBytes[2 * i] & 0xFF) << 8 | (addressBytes[2 * i + 1] & 0xFF);
+ }
+
+ // now, find the sequence of zeros that should be compressed
+ int bestRunStart = -1;
+ int bestRunLength = -1;
+ int runStart = -1;
+ for (int i = 0; i < hextets.length + 1; i++) {
+ if (i < hextets.length && hextets[i] == 0) {
+ if (runStart < 0) {
+ runStart = i;
+ }
+ } else if (runStart >= 0) {
+ int runLength = i - runStart;
+ if (runLength > bestRunLength) {
+ bestRunStart = runStart;
+ bestRunLength = runLength;
+ }
+ runStart = -1;
+ }
+ }
+ if (bestRunLength >= 2) {
+ Arrays.fill(hextets, bestRunStart, bestRunStart + bestRunLength, -1);
+ }
+
+ // convert into text form
+ StringBuilder buf = new StringBuilder(40);
+ buf.append('[');
+
+ boolean lastWasNumber = false;
+ for (int i = 0; i < hextets.length; i++) {
+ boolean thisIsNumber = hextets[i] >= 0;
+ if (thisIsNumber) {
+ if (lastWasNumber) {
+ buf.append(':');
+ }
+ buf.append(Integer.toHexString(hextets[i]));
+ } else {
+ if (i == 0 || lastWasNumber) {
+ buf.append("::");
+ }
+ }
+ lastWasNumber = thisIsNumber;
+ }
+ buf.append(']');
+ return buf.toString();
+ }
+
+ // ------------------------------------------------------------------------
+ // Port range parsing
+ // ------------------------------------------------------------------------
+
+ /**
* Returns an iterator over available ports defined by the range definition.
*
* @param rangeDefinition String describing a single port, a range of ports or multiple ranges.
@@ -186,14 +247,16 @@ public class NetUtils {
*/
public static Iterator<Integer> getPortRangeFromString(String rangeDefinition) throws NumberFormatException {
final String[] ranges = rangeDefinition.trim().split(",");
- List<Iterator<Integer>> iterators = new ArrayList<>(ranges.length);
- for(String rawRange: ranges) {
+
+ UnionIterator<Integer> iterators = new UnionIterator<>();
+
+ for (String rawRange: ranges) {
Iterator<Integer> rangeIterator;
String range = rawRange.trim();
int dashIdx = range.indexOf('-');
if (dashIdx == -1) {
// only one port in range:
- rangeIterator = Iterators.singletonIterator(Integer.valueOf(range));
+ rangeIterator = Collections.singleton(Integer.valueOf(range)).iterator();
} else {
// evaluate range
final int start = Integer.valueOf(range.substring(0, dashIdx));
@@ -218,7 +281,8 @@ public class NetUtils {
}
iterators.add(rangeIterator);
}
- return Iterators.concat(iterators.iterator());
+
+ return iterators;
}
/**
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-core/src/main/java/org/apache/flink/util/Preconditions.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/util/Preconditions.java b/flink-core/src/main/java/org/apache/flink/util/Preconditions.java
index 135038b..a9bd166 100644
--- a/flink-core/src/main/java/org/apache/flink/util/Preconditions.java
+++ b/flink-core/src/main/java/org/apache/flink/util/Preconditions.java
@@ -107,7 +107,7 @@ public final class Preconditions {
}
// ------------------------------------------------------------------------
- // Boolean Condition Checking
+ // Boolean Condition Checking (Argument)
// ------------------------------------------------------------------------
/**
@@ -162,6 +162,61 @@ public final class Preconditions {
}
}
+ // ------------------------------------------------------------------------
+ // Boolean Condition Checking (State)
+ // ------------------------------------------------------------------------
+
+ /**
+ * Checks the given boolean condition, and throws an {@code IllegalStateException} if
+ * the condition is not met (evaluates to {@code false}).
+ *
+ * @param condition The condition to check
+ *
+ * @throws IllegalStateException Thrown, if the condition is violated.
+ */
+ public static void checkState(boolean condition) {
+ if (!condition) {
+ throw new IllegalStateException();
+ }
+ }
+
+ /**
+ * Checks the given boolean condition, and throws an {@code IllegalStateException} if
+ * the condition is not met (evaluates to {@code false}). The exception will have the
+ * given error message.
+ *
+ * @param condition The condition to check
+ * @param errorMessage The message for the {@code IllegalStateException} that is thrown if the check fails.
+ *
+ * @throws IllegalStateException Thrown, if the condition is violated.
+ */
+ public static void checkState(boolean condition, @Nullable Object errorMessage) {
+ if (!condition) {
+ throw new IllegalStateException(String.valueOf(errorMessage));
+ }
+ }
+
+ /**
+ * Checks the given boolean condition, and throws an {@code IllegalStateException} if
+ * the condition is not met (evaluates to {@code false}).
+ *
+ * @param condition The condition to check
+ * @param errorMessageTemplate The message template for the {@code IllegalStateException}
+ * that is thrown if the check fails. The template substitutes its
+ * {@code %s} placeholders with the error message arguments.
+ * @param errorMessageArgs The arguments for the error message, to be inserted into the
+ * message template for the {@code %s} placeholders.
+ *
+ * @throws IllegalStateException Thrown, if the condition is violated.
+ */
+ public static void checkState(boolean condition,
+ @Nullable String errorMessageTemplate,
+ @Nullable Object... errorMessageArgs) {
+
+ if (!condition) {
+ throw new IllegalStateException(format(errorMessageTemplate, errorMessageArgs));
+ }
+ }
// ------------------------------------------------------------------------
// Utilities
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-core/src/main/java/org/apache/flink/util/UnionIterator.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/util/UnionIterator.java b/flink-core/src/main/java/org/apache/flink/util/UnionIterator.java
new file mode 100644
index 0000000..17204ce
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/util/UnionIterator.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.util;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+public class UnionIterator<T> implements Iterator<T>, Iterable<T> {
+
+ private Iterator<T> currentIterator;
+
+ private ArrayList<Iterator<T>> furtherIterators = new ArrayList<>();
+
+ private int nextIterator;
+
+ private boolean iteratorAvailable = true;
+
+ // ------------------------------------------------------------------------
+
+ public void clear() {
+ currentIterator = null;
+ furtherIterators.clear();
+ nextIterator = 0;
+ iteratorAvailable = true;
+ }
+
+ public void addList(List<T> list) {
+ add(list.iterator());
+ }
+
+ public void add(Iterator<T> iterator) {
+ if (currentIterator == null) {
+ currentIterator = iterator;
+ }
+ else {
+ furtherIterators.add(iterator);
+ }
+ }
+
+ // ------------------------------------------------------------------------
+
+ @Override
+ public Iterator<T> iterator() {
+ if (iteratorAvailable) {
+ iteratorAvailable = false;
+ return this;
+ } else {
+ throw new TraversableOnceException();
+ }
+ }
+
+ @Override
+ public boolean hasNext() {
+ while (currentIterator != null) {
+ if (currentIterator.hasNext()) {
+ return true;
+ }
+ else if (nextIterator < furtherIterators.size()) {
+ currentIterator = furtherIterators.get(nextIterator);
+ nextIterator++;
+ }
+ else {
+ currentIterator = null;
+ }
+ }
+
+ return false;
+ }
+
+ @Override
+ public T next() {
+ if (hasNext()) {
+ return currentIterator.next();
+ }
+ else {
+ throw new NoSuchElementException();
+ }
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-core/src/main/java/org/apache/flink/util/XORShiftRandom.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/util/XORShiftRandom.java b/flink-core/src/main/java/org/apache/flink/util/XORShiftRandom.java
index 47ed561..b3ec393 100644
--- a/flink-core/src/main/java/org/apache/flink/util/XORShiftRandom.java
+++ b/flink-core/src/main/java/org/apache/flink/util/XORShiftRandom.java
@@ -15,9 +15,9 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.flink.util;
-import com.google.common.hash.Hashing;
import org.apache.flink.annotation.Public;
import java.util.Random;
@@ -42,7 +42,7 @@ public class XORShiftRandom extends Random {
public XORShiftRandom(long input) {
super(input);
- this.seed = Hashing.murmur3_128().hashLong(input).asLong();
+ this.seed = MathUtils.murmurHash((int) input) ^ MathUtils.murmurHash((int) (input >>> 32));
}
/**
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-core/src/test/java/org/apache/flink/api/common/operators/base/OuterJoinOperatorBaseTest.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/api/common/operators/base/OuterJoinOperatorBaseTest.java b/flink-core/src/test/java/org/apache/flink/api/common/operators/base/OuterJoinOperatorBaseTest.java
index 679e4ce..69159f2 100644
--- a/flink-core/src/test/java/org/apache/flink/api/common/operators/base/OuterJoinOperatorBaseTest.java
+++ b/flink-core/src/test/java/org/apache/flink/api/common/operators/base/OuterJoinOperatorBaseTest.java
@@ -18,12 +18,10 @@
package org.apache.flink.api.common.operators.base;
-
import java.io.Serializable;
import java.util.Arrays;
import java.util.List;
-import com.google.common.base.Joiner;
import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.functions.FlatJoinFunction;
import org.apache.flink.api.common.operators.BinaryOperatorInformation;
@@ -39,7 +37,7 @@ public class OuterJoinOperatorBaseTest implements Serializable {
private final FlatJoinFunction<String, String, String> joiner = new FlatJoinFunction<String, String, String>() {
@Override
public void join(String first, String second, Collector<String> out) throws Exception {
- out.collect(Joiner.on(',').join(String.valueOf(first), String.valueOf(second)));
+ out.collect(String.valueOf(first) + ',' + String.valueOf(second));
}
};
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-core/src/test/java/org/apache/flink/api/java/tuple/TupleGenerator.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/api/java/tuple/TupleGenerator.java b/flink-core/src/test/java/org/apache/flink/api/java/tuple/TupleGenerator.java
index e783a1d..ee37194 100644
--- a/flink-core/src/test/java/org/apache/flink/api/java/tuple/TupleGenerator.java
+++ b/flink-core/src/test/java/org/apache/flink/api/java/tuple/TupleGenerator.java
@@ -18,8 +18,7 @@
package org.apache.flink.api.java.tuple;
-import com.google.common.base.Charsets;
-import com.google.common.io.Files;
+import org.apache.flink.util.FileUtils;
import java.io.File;
import java.io.FileNotFoundException;
@@ -85,7 +84,7 @@ class TupleGenerator {
}
private static void insertCodeIntoFile(String code, File file) throws IOException {
- String fileContent = Files.toString(file, Charsets.UTF_8);
+ String fileContent = FileUtils.readFileUtf8(file);
try (Scanner s = new Scanner(fileContent)) {
StringBuilder sb = new StringBuilder();
@@ -126,7 +125,7 @@ class TupleGenerator {
while (s.hasNextLine() && (line = s.nextLine()) != null) {
sb.append(line).append("\n");
}
- Files.write(sb.toString(), file, Charsets.UTF_8);
+ FileUtils.writeFileUtf8(file, sb.toString());
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-core/src/test/java/org/apache/flink/api/java/typeutils/PojoTypeExtractionTest.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/PojoTypeExtractionTest.java b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/PojoTypeExtractionTest.java
index bc11848..ffcfd52 100644
--- a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/PojoTypeExtractionTest.java
+++ b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/PojoTypeExtractionTest.java
@@ -22,6 +22,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Date;
+import java.util.HashSet;
import java.util.List;
import org.apache.flink.api.common.functions.MapFunction;
@@ -37,8 +38,6 @@ import org.apache.flink.api.java.typeutils.TypeInfoParserTest.MyWritable;
import org.junit.Assert;
import org.junit.Test;
-import com.google.common.collect.HashMultiset;
-
/**
* Pojo Type tests
*
@@ -87,7 +86,7 @@ public class PojoTypeExtractionTest {
// all public test
public static class AllPublic extends ComplexNestedClass {
public ArrayList<String> somethingFancy; // generic type
- public HashMultiset<Integer> fancyIds; // generic type
+ public FancyCollectionSubtype<Integer> fancyIds; // generic type
public String[] fancyArray; // generic type
}
@@ -436,7 +435,7 @@ public class PojoTypeExtractionTest {
}
multisetSeen = true;
Assert.assertTrue(field.getTypeInformation() instanceof GenericTypeInfo);
- Assert.assertEquals(HashMultiset.class, field.getTypeInformation().getTypeClass());
+ Assert.assertEquals(FancyCollectionSubtype.class, field.getTypeInformation().getTypeClass());
} else if(name.equals("fancyArray")) {
if(strArraySeen) {
Assert.fail("already seen");
@@ -809,4 +808,10 @@ public class PojoTypeExtractionTest {
Assert.assertTrue(tti.getTypeAt(0) instanceof PojoTypeInfo);
Assert.assertTrue(tti.getTypeAt(1) instanceof PojoTypeInfo);
}
+
+ // ------------------------------------------------------------------------
+
+ public static class FancyCollectionSubtype<T> extends HashSet<T> {
+ private static final long serialVersionUID = -3494469602638179921L;
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerTest.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerTest.java b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerTest.java
index 260f7e9..4712ed1 100644
--- a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerTest.java
+++ b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerTest.java
@@ -21,6 +21,7 @@ package org.apache.flink.api.java.typeutils.runtime;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
+import java.util.Objects;
import java.util.Random;
import org.apache.flink.api.common.ExecutionConfig;
@@ -36,11 +37,10 @@ import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.api.java.typeutils.PojoTypeInfo;
import org.apache.flink.api.java.typeutils.TupleTypeInfo;
import org.apache.flink.api.java.typeutils.TypeExtractor;
+
import org.junit.Assert;
import org.junit.Test;
-import com.google.common.base.Objects;
-
/**
* A test for the {@link PojoSerializer}.
*/
@@ -104,7 +104,7 @@ public class PojoSerializerTest extends SerializerTestBase<PojoSerializerTest.Te
@Override
public int hashCode() {
- return Objects.hashCode(dumm1, dumm2, dumm3, dumm4, nestedClass);
+ return Objects.hash(dumm1, dumm2, dumm3, dumm4, nestedClass);
}
@Override
@@ -162,7 +162,7 @@ public class PojoSerializerTest extends SerializerTestBase<PojoSerializerTest.Te
@Override
public int hashCode() {
- return Objects.hashCode(dumm1, dumm2, dumm3, dumm4);
+ return Objects.hash(dumm1, dumm2, dumm3, dumm4);
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoSubclassSerializerTest.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoSubclassSerializerTest.java b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoSubclassSerializerTest.java
index 8c61a19..ddf1d0e 100644
--- a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoSubclassSerializerTest.java
+++ b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoSubclassSerializerTest.java
@@ -18,14 +18,15 @@
package org.apache.flink.api.java.typeutils.runtime;
-import com.google.common.base.Objects;
import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.common.typeutils.SerializerTestBase;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.java.typeutils.TypeExtractor;
+
import org.junit.Test;
+import java.util.Objects;
import java.util.Random;
/**
@@ -88,7 +89,7 @@ public class PojoSubclassSerializerTest extends SerializerTestBase<PojoSubclassS
@Override
public int hashCode() {
- return Objects.hashCode(dumm1, dumm2);
+ return Objects.hash(dumm1, dumm2);
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/SubclassFromInterfaceSerializerTest.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/SubclassFromInterfaceSerializerTest.java b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/SubclassFromInterfaceSerializerTest.java
index b797090..e6ffd07 100644
--- a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/SubclassFromInterfaceSerializerTest.java
+++ b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/SubclassFromInterfaceSerializerTest.java
@@ -18,15 +18,16 @@
package org.apache.flink.api.java.typeutils.runtime;
-import com.google.common.base.Objects;
import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.common.typeutils.SerializerTestBase;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.java.typeutils.TypeExtractor;
import org.apache.flink.api.java.typeutils.runtime.kryo.KryoSerializer;
+
import org.junit.Test;
+import java.util.Objects;
import java.util.Random;
/**
@@ -90,7 +91,7 @@ public class SubclassFromInterfaceSerializerTest extends SerializerTestBase<Subc
@Override
public int hashCode() {
- return Objects.hashCode(dumm1, dumm2);
+ return Objects.hash(dumm1, dumm2);
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-core/src/test/java/org/apache/flink/testutils/junit/RetryRule.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/testutils/junit/RetryRule.java b/flink-core/src/test/java/org/apache/flink/testutils/junit/RetryRule.java
index 2b3a37a..782f4fb 100644
--- a/flink-core/src/test/java/org/apache/flink/testutils/junit/RetryRule.java
+++ b/flink-core/src/test/java/org/apache/flink/testutils/junit/RetryRule.java
@@ -22,11 +22,12 @@ import org.junit.Test;
import org.junit.rules.TestRule;
import org.junit.runner.Description;
import org.junit.runners.model.Statement;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkNotNull;
+import static org.apache.flink.util.Preconditions.checkArgument;
+import static org.apache.flink.util.Preconditions.checkNotNull;
/**
* A rule to retry failed tests for a fixed number of times.
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-core/src/test/java/org/apache/flink/util/MathUtilTest.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/util/MathUtilTest.java b/flink-core/src/test/java/org/apache/flink/util/MathUtilTest.java
new file mode 100644
index 0000000..7917a7b
--- /dev/null
+++ b/flink-core/src/test/java/org/apache/flink/util/MathUtilTest.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.util;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.fail;
+
+import org.apache.flink.util.MathUtils;
+import org.junit.Test;
+
+public class MathUtilTest {
+
+ @Test
+ public void testLog2Computation() {
+ assertEquals(0, MathUtils.log2floor(1));
+ assertEquals(1, MathUtils.log2floor(2));
+ assertEquals(1, MathUtils.log2floor(3));
+ assertEquals(2, MathUtils.log2floor(4));
+ assertEquals(2, MathUtils.log2floor(5));
+ assertEquals(2, MathUtils.log2floor(7));
+ assertEquals(3, MathUtils.log2floor(8));
+ assertEquals(3, MathUtils.log2floor(9));
+ assertEquals(4, MathUtils.log2floor(16));
+ assertEquals(4, MathUtils.log2floor(17));
+ assertEquals(13, MathUtils.log2floor((0x1 << 13) + 1));
+ assertEquals(30, MathUtils.log2floor(Integer.MAX_VALUE));
+ assertEquals(31, MathUtils.log2floor(-1));
+
+ try {
+ MathUtils.log2floor(0);
+ fail();
+ }
+ catch (ArithmeticException aex) {}
+ }
+
+ @Test
+ public void testRoundDownToPowerOf2() {
+ assertEquals(0, MathUtils.roundDownToPowerOf2(0));
+ assertEquals(1, MathUtils.roundDownToPowerOf2(1));
+ assertEquals(2, MathUtils.roundDownToPowerOf2(2));
+ assertEquals(2, MathUtils.roundDownToPowerOf2(3));
+ assertEquals(4, MathUtils.roundDownToPowerOf2(4));
+ assertEquals(4, MathUtils.roundDownToPowerOf2(5));
+ assertEquals(4, MathUtils.roundDownToPowerOf2(6));
+ assertEquals(4, MathUtils.roundDownToPowerOf2(7));
+ assertEquals(8, MathUtils.roundDownToPowerOf2(8));
+ assertEquals(8, MathUtils.roundDownToPowerOf2(9));
+ assertEquals(8, MathUtils.roundDownToPowerOf2(15));
+ assertEquals(16, MathUtils.roundDownToPowerOf2(16));
+ assertEquals(16, MathUtils.roundDownToPowerOf2(17));
+ assertEquals(16, MathUtils.roundDownToPowerOf2(31));
+ assertEquals(32, MathUtils.roundDownToPowerOf2(32));
+ assertEquals(32, MathUtils.roundDownToPowerOf2(33));
+ assertEquals(32, MathUtils.roundDownToPowerOf2(42));
+ assertEquals(32, MathUtils.roundDownToPowerOf2(63));
+ assertEquals(64, MathUtils.roundDownToPowerOf2(64));
+ assertEquals(64, MathUtils.roundDownToPowerOf2(125));
+ assertEquals(16384, MathUtils.roundDownToPowerOf2(25654));
+ assertEquals(33554432, MathUtils.roundDownToPowerOf2(34366363));
+ assertEquals(33554432, MathUtils.roundDownToPowerOf2(63463463));
+ assertEquals(1073741824, MathUtils.roundDownToPowerOf2(1852987883));
+ assertEquals(1073741824, MathUtils.roundDownToPowerOf2(Integer.MAX_VALUE));
+ }
+
+ @Test
+ public void testPowerOfTwo() {
+ assertTrue(MathUtils.isPowerOf2(1));
+ assertTrue(MathUtils.isPowerOf2(2));
+ assertTrue(MathUtils.isPowerOf2(4));
+ assertTrue(MathUtils.isPowerOf2(8));
+ assertTrue(MathUtils.isPowerOf2(32768));
+ assertTrue(MathUtils.isPowerOf2(65536));
+ assertTrue(MathUtils.isPowerOf2(1 << 30));
+ assertTrue(MathUtils.isPowerOf2(1L + Integer.MAX_VALUE));
+ assertTrue(MathUtils.isPowerOf2(1L << 41));
+ assertTrue(MathUtils.isPowerOf2(1L << 62));
+
+ assertFalse(MathUtils.isPowerOf2(3));
+ assertFalse(MathUtils.isPowerOf2(5));
+ assertFalse(MathUtils.isPowerOf2(567923));
+ assertFalse(MathUtils.isPowerOf2(Integer.MAX_VALUE));
+ assertFalse(MathUtils.isPowerOf2(Long.MAX_VALUE));
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-core/src/test/java/org/apache/flink/util/NetUtilsTest.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/util/NetUtilsTest.java b/flink-core/src/test/java/org/apache/flink/util/NetUtilsTest.java
index e367e8b..72cc89e 100644
--- a/flink-core/src/test/java/org/apache/flink/util/NetUtilsTest.java
+++ b/flink-core/src/test/java/org/apache/flink/util/NetUtilsTest.java
@@ -18,17 +18,13 @@
package org.apache.flink.util;
-import com.google.common.collect.DiscreteDomain;
-import com.google.common.collect.Iterators;
import org.junit.Assert;
import org.junit.Test;
import java.net.InetAddress;
import java.net.InetSocketAddress;
-import java.util.ArrayList;
import java.util.HashSet;
import java.util.Iterator;
-import java.util.List;
import java.util.Set;
import static org.hamcrest.core.IsCollectionContaining.hasItems;
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-core/src/test/java/org/apache/flink/util/UnionIteratorTest.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/util/UnionIteratorTest.java b/flink-core/src/test/java/org/apache/flink/util/UnionIteratorTest.java
new file mode 100644
index 0000000..4c1fc41
--- /dev/null
+++ b/flink-core/src/test/java/org/apache/flink/util/UnionIteratorTest.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.util;
+
+import org.apache.flink.util.TraversableOnceException;
+import org.apache.flink.util.UnionIterator;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+
+import static org.junit.Assert.*;
+
+public class UnionIteratorTest {
+
+ @Test
+ public void testUnion() {
+ try {
+ UnionIterator<Integer> iter = new UnionIterator<>();
+
+ // should succeed and be empty
+ assertFalse(iter.iterator().hasNext());
+
+ iter.clear();
+
+ try {
+ iter.iterator().next();
+ fail("should fail with an exception");
+ } catch (NoSuchElementException e) {
+ // expected
+ }
+
+ iter.clear();
+ iter.addList(Arrays.asList(1, 2, 3, 4, 5, 6, 7));
+ iter.addList(Collections.<Integer>emptyList());
+ iter.addList(Arrays.asList(8, 9, 10, 11));
+
+ int val = 1;
+ for (int i : iter) {
+ assertEquals(val++, i);
+ }
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testTraversableOnce() {
+ try {
+ UnionIterator<Integer> iter = new UnionIterator<>();
+
+ // should succeed
+ iter.iterator();
+
+ // should fail
+ try {
+ iter.iterator();
+ fail("should fail with an exception");
+ } catch (TraversableOnceException e) {
+ // expected
+ }
+
+ // should fail again
+ try {
+ iter.iterator();
+ fail("should fail with an exception");
+ } catch (TraversableOnceException e) {
+ // expected
+ }
+
+ // reset the thing, keep it empty
+ iter.clear();
+
+ // should succeed
+ iter.iterator();
+
+ // should fail
+ try {
+ iter.iterator();
+ fail("should fail with an exception");
+ } catch (TraversableOnceException e) {
+ // expected
+ }
+
+ // should fail again
+ try {
+ iter.iterator();
+ fail("should fail with an exception");
+ } catch (TraversableOnceException e) {
+ // expected
+ }
+
+ // reset the thing, add some data
+ iter.clear();
+ iter.addList(Arrays.asList(1, 2, 3, 4, 5, 6, 7));
+
+ // should succeed
+ Iterator<Integer> ints = iter.iterator();
+ assertNotNull(ints.next());
+ assertNotNull(ints.next());
+ assertNotNull(ints.next());
+
+ // should fail if called in the middle of operations
+ try {
+ iter.iterator();
+ fail("should fail with an exception");
+ } catch (TraversableOnceException e) {
+ // expected
+ }
+
+ // reset the thing, keep it empty
+ iter.clear();
+
+ // should succeed again
+ assertFalse(iter.iterator().hasNext());
+
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-runtime/src/main/java/org/apache/flink/runtime/blob/BlobUtils.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/blob/BlobUtils.java b/flink-runtime/src/main/java/org/apache/flink/runtime/blob/BlobUtils.java
index d8f744b..99237a3 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/blob/BlobUtils.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/blob/BlobUtils.java
@@ -23,7 +23,7 @@ import org.apache.commons.io.FileUtils;
import org.apache.flink.api.common.JobID;
import org.apache.flink.core.fs.FileSystem;
import org.apache.flink.core.fs.Path;
-import org.apache.flink.runtime.util.IOUtils;
+import org.apache.flink.util.IOUtils;
import org.slf4j.Logger;
import java.io.EOFException;
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-runtime/src/main/java/org/apache/flink/runtime/blob/FileSystemBlobStore.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/blob/FileSystemBlobStore.java b/flink-runtime/src/main/java/org/apache/flink/runtime/blob/FileSystemBlobStore.java
index efdb003..5f65564 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/blob/FileSystemBlobStore.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/blob/FileSystemBlobStore.java
@@ -25,7 +25,7 @@ import org.apache.flink.configuration.Configuration;
import org.apache.flink.configuration.IllegalConfigurationException;
import org.apache.flink.core.fs.FileSystem;
import org.apache.flink.core.fs.Path;
-import org.apache.flink.runtime.util.IOUtils;
+import org.apache.flink.util.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/FileSystemStateStore.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/FileSystemStateStore.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/FileSystemStateStore.java
index 8d9d3a7..73a094a 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/FileSystemStateStore.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/FileSystemStateStore.java
@@ -22,7 +22,7 @@ import org.apache.flink.core.fs.FSDataOutputStream;
import org.apache.flink.core.fs.FileSystem;
import org.apache.flink.core.fs.Path;
import org.apache.flink.runtime.state.filesystem.FileSerializableStateHandle;
-import org.apache.flink.runtime.util.FileUtils;
+import org.apache.flink.util.FileUtils;
import java.io.IOException;
import java.io.ObjectOutputStream;
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-runtime/src/main/java/org/apache/flink/runtime/filecache/FileCache.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/filecache/FileCache.java b/flink-runtime/src/main/java/org/apache/flink/runtime/filecache/FileCache.java
index 1b8b1d9..b9ee2c8 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/filecache/FileCache.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/filecache/FileCache.java
@@ -42,7 +42,7 @@ import org.apache.flink.core.fs.FileSystem;
import org.apache.flink.core.fs.Path;
import org.apache.flink.api.common.JobID;
import org.apache.flink.runtime.util.ExecutorThreadFactory;
-import org.apache.flink.runtime.util.IOUtils;
+import org.apache.flink.util.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-runtime/src/main/java/org/apache/flink/runtime/io/disk/FileChannelInputView.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/disk/FileChannelInputView.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/disk/FileChannelInputView.java
index 2ca7f78..992631b 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/disk/FileChannelInputView.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/disk/FileChannelInputView.java
@@ -29,7 +29,7 @@ import org.apache.flink.core.memory.MemorySegment;
import org.apache.flink.runtime.io.disk.iomanager.BlockChannelReader;
import org.apache.flink.runtime.memory.AbstractPagedInputView;
import org.apache.flink.runtime.memory.MemoryManager;
-import org.apache.flink.runtime.util.MathUtils;
+import org.apache.flink.util.MathUtils;
/**
* A {@link org.apache.flink.core.memory.DataInputView} that is backed by a {@link BlockChannelReader},
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-runtime/src/main/java/org/apache/flink/runtime/io/disk/RandomAccessInputView.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/disk/RandomAccessInputView.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/disk/RandomAccessInputView.java
index 23dccb0..2ff6c94 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/disk/RandomAccessInputView.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/disk/RandomAccessInputView.java
@@ -24,7 +24,7 @@ import java.util.ArrayList;
import org.apache.flink.core.memory.MemorySegment;
import org.apache.flink.core.memory.SeekableDataInputView;
import org.apache.flink.runtime.memory.AbstractPagedInputView;
-import org.apache.flink.runtime.util.MathUtils;
+import org.apache.flink.util.MathUtils;
public class RandomAccessInputView extends AbstractPagedInputView implements SeekableDataInputView {
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-runtime/src/main/java/org/apache/flink/runtime/io/disk/RandomAccessOutputView.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/disk/RandomAccessOutputView.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/disk/RandomAccessOutputView.java
index 427fe84..b0ad184 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/disk/RandomAccessOutputView.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/disk/RandomAccessOutputView.java
@@ -24,7 +24,7 @@ import java.io.EOFException;
import org.apache.flink.core.memory.MemorySegment;
import org.apache.flink.core.memory.SeekableDataOutputView;
import org.apache.flink.runtime.memory.AbstractPagedOutputView;
-import org.apache.flink.runtime.util.MathUtils;
+import org.apache.flink.util.MathUtils;
/**
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-runtime/src/main/java/org/apache/flink/runtime/io/disk/SeekableFileChannelInputView.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/disk/SeekableFileChannelInputView.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/disk/SeekableFileChannelInputView.java
index 355b2eb..e768c77 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/disk/SeekableFileChannelInputView.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/disk/SeekableFileChannelInputView.java
@@ -30,7 +30,7 @@ import org.apache.flink.runtime.io.disk.iomanager.FileIOChannel;
import org.apache.flink.runtime.io.disk.iomanager.IOManager;
import org.apache.flink.runtime.memory.AbstractPagedInputView;
import org.apache.flink.runtime.memory.MemoryManager;
-import org.apache.flink.runtime.util.MathUtils;
+import org.apache.flink.util.MathUtils;
/**
* A {@link org.apache.flink.core.memory.DataInputView} that is backed by a {@link BlockChannelReader},
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-runtime/src/main/java/org/apache/flink/runtime/io/disk/SimpleCollectingOutputView.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/disk/SimpleCollectingOutputView.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/disk/SimpleCollectingOutputView.java
index a107e79..1a45ff2 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/disk/SimpleCollectingOutputView.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/disk/SimpleCollectingOutputView.java
@@ -26,7 +26,7 @@ import java.util.List;
import org.apache.flink.core.memory.MemorySegment;
import org.apache.flink.core.memory.MemorySegmentSource;
import org.apache.flink.runtime.memory.AbstractPagedOutputView;
-import org.apache.flink.runtime.util.MathUtils;
+import org.apache.flink.util.MathUtils;
/**
* The list with the full segments contains at any point all completely full segments, plus the segment that is
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-runtime/src/main/java/org/apache/flink/runtime/memory/MemoryManager.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/memory/MemoryManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/memory/MemoryManager.java
index 094d065..3ade753 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/memory/MemoryManager.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/memory/MemoryManager.java
@@ -33,7 +33,7 @@ import org.apache.flink.core.memory.HeapMemorySegment;
import org.apache.flink.core.memory.HybridMemorySegment;
import org.apache.flink.core.memory.MemorySegment;
import org.apache.flink.core.memory.MemoryType;
-import org.apache.flink.runtime.util.MathUtils;
+import org.apache.flink.util.MathUtils;
/**
* The memory manager governs the memory that Flink uses for sorting, hashing, and caching. Memory
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-runtime/src/main/java/org/apache/flink/runtime/operators/hash/CompactingHashTable.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/hash/CompactingHashTable.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/hash/CompactingHashTable.java
index b4d03e7..0ff2ce8 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/hash/CompactingHashTable.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/hash/CompactingHashTable.java
@@ -33,7 +33,7 @@ import org.apache.flink.core.memory.MemorySegment;
import org.apache.flink.runtime.memory.ListMemorySegmentSource;
import org.apache.flink.runtime.util.IntArrayList;
import org.apache.flink.runtime.util.LongArrayList;
-import org.apache.flink.runtime.util.MathUtils;
+import org.apache.flink.util.MathUtils;
import org.apache.flink.util.MutableObjectIterator;
/**
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-runtime/src/main/java/org/apache/flink/runtime/operators/hash/HashPartition.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/hash/HashPartition.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/hash/HashPartition.java
index 44ee163..9221e7e 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/hash/HashPartition.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/hash/HashPartition.java
@@ -37,7 +37,7 @@ import org.apache.flink.runtime.io.disk.iomanager.ChannelWriterOutputView;
import org.apache.flink.runtime.io.disk.iomanager.IOManager;
import org.apache.flink.runtime.memory.AbstractPagedInputView;
import org.apache.flink.runtime.memory.AbstractPagedOutputView;
-import org.apache.flink.runtime.util.MathUtils;
+import org.apache.flink.util.MathUtils;
import org.apache.flink.util.MutableObjectIterator;
/**
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-runtime/src/main/java/org/apache/flink/runtime/operators/hash/MutableHashTable.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/hash/MutableHashTable.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/hash/MutableHashTable.java
index 1495ee1..351cd3f 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/hash/MutableHashTable.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/hash/MutableHashTable.java
@@ -42,7 +42,7 @@ import org.apache.flink.runtime.io.disk.iomanager.ChannelReaderInputView;
import org.apache.flink.runtime.io.disk.iomanager.HeaderlessChannelReaderInputView;
import org.apache.flink.runtime.io.disk.iomanager.IOManager;
import org.apache.flink.runtime.operators.util.BloomFilter;
-import org.apache.flink.runtime.util.MathUtils;
+import org.apache.flink.util.MathUtils;
import org.apache.flink.util.MutableObjectIterator;
/**
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-runtime/src/main/java/org/apache/flink/runtime/operators/shipping/OutputEmitter.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/shipping/OutputEmitter.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/shipping/OutputEmitter.java
index fdbcd9f..4e20842 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/shipping/OutputEmitter.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/shipping/OutputEmitter.java
@@ -23,7 +23,7 @@ import org.apache.flink.api.common.functions.Partitioner;
import org.apache.flink.api.common.typeutils.TypeComparator;
import org.apache.flink.runtime.io.network.api.writer.ChannelSelector;
import org.apache.flink.runtime.plugable.SerializationDelegate;
-import org.apache.flink.runtime.util.MathUtils;
+import org.apache.flink.util.MathUtils;
/**
* The output emitter decides to which of the possibly multiple output channels a record is sent.
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-runtime/src/main/java/org/apache/flink/runtime/util/FileUtils.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/util/FileUtils.java b/flink-runtime/src/main/java/org/apache/flink/runtime/util/FileUtils.java
deleted file mode 100644
index 42994d3..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/util/FileUtils.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.util;
-
-/**
- * This is a utility class to deal with temporary files.
- */
-public final class FileUtils {
-
- /**
- * The alphabet to construct the random part of the filename from.
- */
- private static final char[] ALPHABET = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '0', 'a', 'b', 'c', 'd',
- 'e', 'f' };
-
- /**
- * The length of the random part of the filename.
- */
- private static final int LENGTH = 12;
-
- /**
- * Empty private constructor to avoid instantiation.
- */
- private FileUtils() {
- }
-
- /**
- * Constructs a random filename with the given prefix and
- * a random part generated from hex characters.
- *
- * @param prefix
- * the prefix to the filename to be constructed
- * @return the generated random filename with the given prefix
- */
- public static String getRandomFilename(final String prefix) {
-
- final StringBuilder stringBuilder = new StringBuilder(prefix);
-
- for (int i = 0; i < LENGTH; i++) {
- stringBuilder.append(ALPHABET[(int) Math.floor(Math.random() * (double) ALPHABET.length)]);
- }
-
- return stringBuilder.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-runtime/src/main/java/org/apache/flink/runtime/util/IOUtils.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/util/IOUtils.java b/flink-runtime/src/main/java/org/apache/flink/runtime/util/IOUtils.java
deleted file mode 100644
index d03b72c..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/util/IOUtils.java
+++ /dev/null
@@ -1,213 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.runtime.util;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.io.PrintStream;
-import java.net.Socket;
-
-import org.slf4j.Logger;
-
-/**
- * An utility class for I/O related functionality.
- *
- */
-public final class IOUtils {
-
- /**
- * The block size for byte operations in byte.
- */
- private static final int BLOCKSIZE = 4096;
-
- /**
- * Private constructor to overwrite the public one.
- */
- private IOUtils() {
- }
-
- /**
- * Copies from one stream to another.
- *
- * @param in
- * InputStream to read from
- * @param out
- * OutputStream to write to
- * @param buffSize
- * the size of the buffer
- * @param close
- * whether or not close the InputStream and OutputStream at the end. The streams are closed in the finally
- * clause.
- * @throws IOException
- * thrown if an error occurred while writing to the output stream
- */
- public static void copyBytes(final InputStream in, final OutputStream out, final int buffSize, final boolean close)
- throws IOException {
-
- @SuppressWarnings("resource")
- final PrintStream ps = out instanceof PrintStream ? (PrintStream) out : null;
- final byte[] buf = new byte[buffSize];
- try {
- int bytesRead = in.read(buf);
- while (bytesRead >= 0) {
- out.write(buf, 0, bytesRead);
- if ((ps != null) && ps.checkError()) {
- throw new IOException("Unable to write to output stream.");
- }
- bytesRead = in.read(buf);
- }
- } finally {
- if (close) {
- out.close();
- in.close();
- }
- }
- }
-
- /**
- * Copies from one stream to another. <strong>closes the input and output
- * streams at the end</strong>.
- *
- * @param in
- * InputStream to read from
- * @param out
- * OutputStream to write to
- * @throws IOException
- * thrown if an I/O error occurs while copying
- */
- public static void copyBytes(final InputStream in, final OutputStream out) throws IOException {
- copyBytes(in, out, BLOCKSIZE, true);
- }
-
- /**
- * Copies from one stream to another.
- *
- * @param in
- * InputStream to read from
- * @param out
- * OutputStream to write to
- * @param close
- * whether or not close the InputStream and OutputStream at the
- * end. The streams are closed in the finally clause.
- * @throws IOException
- * thrown if an I/O error occurs while copying
- */
- public static void copyBytes(final InputStream in, final OutputStream out, final boolean close) throws IOException {
- copyBytes(in, out, BLOCKSIZE, close);
- }
-
- /**
- * Reads len bytes in a loop.
- *
- * @param in
- * The InputStream to read from
- * @param buf
- * The buffer to fill
- * @param off
- * offset from the buffer
- * @param len
- * the length of bytes to read
- * @throws IOException
- * if it could not read requested number of bytes for any reason (including EOF)
- */
- public static void readFully(final InputStream in, final byte[] buf, int off, final int len)
- throws IOException {
- int toRead = len;
- while (toRead > 0) {
- final int ret = in.read(buf, off, toRead);
- if (ret < 0) {
- throw new IOException("Premeture EOF from inputStream");
- }
- toRead -= ret;
- off += ret;
- }
- }
-
- /**
- * Similar to readFully(). Skips bytes in a loop.
- *
- * @param in
- * The InputStream to skip bytes from
- * @param len
- * number of bytes to skip
- * @throws IOException
- * if it could not skip requested number of bytes for any reason (including EOF)
- */
- public static void skipFully(final InputStream in, long len) throws IOException {
- while (len > 0) {
- final long ret = in.skip(len);
- if (ret < 0) {
- throw new IOException("Premeture EOF from inputStream");
- }
- len -= ret;
- }
- }
-
- /**
- * Close the Closeable objects and <b>ignore</b> any {@link IOException} or
- * null pointers. Must only be used for cleanup in exception handlers.
- *
- * @param log
- * the log to record problems to at debug level. Can be <code>null</code>.
- * @param closeables
- * the objects to close
- */
- public static void cleanup(final Logger log, final java.io.Closeable... closeables) {
- for (java.io.Closeable c : closeables) {
- if (c != null) {
- try {
- c.close();
- } catch (IOException e) {
- if (log != null && log.isDebugEnabled()) {
- log.debug("Exception in closing " + c, e);
- }
- }
- }
- }
- }
-
- /**
- * Closes the stream ignoring {@link IOException}. Must only be called in
- * cleaning up from exception handlers.
- *
- * @param stream
- * the stream to close
- */
- public static void closeStream(final java.io.Closeable stream) {
- cleanup(null, stream);
- }
-
- /**
- * Closes the socket ignoring {@link IOException}.
- *
- * @param sock
- * the socket to close
- */
- public static void closeSocket(final Socket sock) {
- // avoids try { close() } dance
- if (sock != null) {
- try {
- sock.close();
- } catch (IOException ignored) {
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-runtime/src/main/java/org/apache/flink/runtime/util/MathUtils.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/util/MathUtils.java b/flink-runtime/src/main/java/org/apache/flink/runtime/util/MathUtils.java
deleted file mode 100644
index 5d26186..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/util/MathUtils.java
+++ /dev/null
@@ -1,176 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.runtime.util;
-
-/**
- * Collection of simple mathematical routines.
- */
-public final class MathUtils {
-
- /**
- * Computes the logarithm of the given value to the base of 2, rounded down. It corresponds to the
- * position of the highest non-zero bit. The position is counted, starting with 0 from the least
- * significant bit to the most significant bit. For example, <code>log2floor(16) = 4</code>, and
- * <code>log2floor(10) = 3</code>.
- *
- * @param value The value to compute the logarithm for.
- * @return The logarithm (rounded down) to the base of 2.
- * @throws ArithmeticException Thrown, if the given value is zero.
- */
- public static int log2floor(int value) throws ArithmeticException {
- if (value == 0) {
- throw new ArithmeticException("Logarithm of zero is undefined.");
- }
-
- int log = 0;
- while ((value = value >>> 1) != 0) {
- log++;
- }
-
- return log;
- }
-
- /**
- * Computes the logarithm of the given value to the base of 2. This method throws an error,
- * if the given argument is not a power of 2.
- *
- * @param value The value to compute the logarithm for.
- * @return The logarithm to the base of 2.
- * @throws ArithmeticException Thrown, if the given value is zero.
- * @throws IllegalArgumentException Thrown, if the given value is not a power of two.
- */
- public static int log2strict(int value) throws ArithmeticException, IllegalArgumentException {
- if (value == 0) {
- throw new ArithmeticException("Logarithm of zero is undefined.");
- }
- if ((value & (value - 1)) != 0) {
- throw new IllegalArgumentException("The given value " + value + " is not a power of two.");
- }
-
- int log = 0;
- while ((value = value >>> 1) != 0) {
- log++;
- }
-
- return log;
- }
-
- /**
- * Decrements the given number down to the closest power of two. If the argument is a
- * power of two, it remains unchanged.
- *
- * @param value The value to round down.
- * @return The closest value that is a power of to and less or equal than the given value.
- */
- public static int roundDownToPowerOf2(int value) {
- return Integer.highestOneBit(value);
- }
-
- /**
- * Casts the given value to a 32 bit integer, if it can be safely done. If the cast would change the numeric
- * value, this method raises an exception.
- * <p>
- * This method is a protection in places where one expects to be able to safely case, but where unexpected
- * situations could make the cast unsafe and would cause hidden problems that are hard to track down.
- *
- * @param value The value to be cast to an integer.
- * @return The given value as an integer.
- */
- public static int checkedDownCast(long value) {
- if (value > Integer.MAX_VALUE) {
- throw new IllegalArgumentException("Cannot downcast long value " + value + " to integer.");
- }
- return (int) value;
- }
-
- /**
- * Checks whether the given value is a power of two.
- *
- * @param value The value to check.
- * @return True, if the value is a power of two, false otherwise.
- */
- public static boolean isPowerOf2(long value) {
- return (value & (value - 1)) == 0;
- }
-
- /**
- * This function hashes an integer value. It is adapted from Bob Jenkins' website
- * <a href="http://www.burtleburtle.net/bob/hash/integer.html">http://www.burtleburtle.net/bob/hash/integer.html</a>.
- * The hash function has the <i>full avalanche</i> property, meaning that every bit of the value to be hashed
- * affects every bit of the hash value.
- *
- * It is crucial to use different hash functions to partition data across machines and the internal partitioning of
- * data structures. This hash function is intended for partitioning internally in data structures.
- *
- * @param code The integer to be hashed.
- * @return The non-negative hash code for the integer.
- */
- public static int jenkinsHash(int code) {
- code = (code + 0x7ed55d16) + (code << 12);
- code = (code ^ 0xc761c23c) ^ (code >>> 19);
- code = (code + 0x165667b1) + (code << 5);
- code = (code + 0xd3a2646c) ^ (code << 9);
- code = (code + 0xfd7046c5) + (code << 3);
- code = (code ^ 0xb55a4f09) ^ (code >>> 16);
- return code >= 0 ? code : -(code + 1);
- }
-
- /**
- * This function hashes an integer value.
- *
- * It is crucial to use different hash functions to partition data across machines and the internal partitioning of
- * data structures. This hash function is intended for partitioning across machines.
- *
- * @param code The integer to be hashed.
- * @return The non-negative hash code for the integer.
- */
- public static int murmurHash(int code) {
- code *= 0xcc9e2d51;
- code = Integer.rotateLeft(code, 15);
- code *= 0x1b873593;
-
- code = Integer.rotateLeft(code, 13);
- code = code * 5 + 0xe6546b64;
-
- code ^= 4;
- code ^= code >>> 16;
- code *= 0x85ebca6b;
- code ^= code >>> 13;
- code *= 0xc2b2ae35;
- code ^= code >>> 16;
-
- if (code >= 0) {
- return code;
- }
- else if (code != Integer.MIN_VALUE) {
- return -code;
- }
- else {
- return 0;
- }
- }
-
- // ============================================================================================
-
- /**
- * Prevent Instantiation through private constructor.
- */
- private MathUtils() {}
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-runtime/src/main/java/org/apache/flink/runtime/util/UnionIterator.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/util/UnionIterator.java b/flink-runtime/src/main/java/org/apache/flink/runtime/util/UnionIterator.java
deleted file mode 100644
index c279adf..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/util/UnionIterator.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.util;
-
-import org.apache.flink.util.TraversableOnceException;
-
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.NoSuchElementException;
-
-public class UnionIterator<T> implements Iterator<T>, Iterable<T> {
-
- private Iterator<T> currentIterator;
-
- private ArrayList<List<T>> furtherLists = new ArrayList<>();
-
- private int nextList;
-
- private boolean iteratorAvailable = true;
-
- // ------------------------------------------------------------------------
-
- public void clear() {
- currentIterator = null;
- furtherLists.clear();
- nextList = 0;
- iteratorAvailable = true;
- }
-
- public void addList(List<T> list) {
- if (currentIterator == null) {
- currentIterator = list.iterator();
- }
- else {
- furtherLists.add(list);
- }
- }
-
- // ------------------------------------------------------------------------
-
- @Override
- public Iterator<T> iterator() {
- if (iteratorAvailable) {
- iteratorAvailable = false;
- return this;
- } else {
- throw new TraversableOnceException();
- }
- }
-
- @Override
- public boolean hasNext() {
- while (currentIterator != null) {
- if (currentIterator.hasNext()) {
- return true;
- }
- else if (nextList < furtherLists.size()) {
- currentIterator = furtherLists.get(nextList).iterator();
- nextList++;
- }
- else {
- currentIterator = null;
- }
- }
-
- return false;
- }
-
- @Override
- public T next() {
- if (hasNext()) {
- return currentIterator.next();
- }
- else {
- throw new NoSuchElementException();
- }
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-runtime/src/main/java/org/apache/flink/runtime/zookeeper/filesystem/FileSystemStateStorageHelper.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/zookeeper/filesystem/FileSystemStateStorageHelper.java b/flink-runtime/src/main/java/org/apache/flink/runtime/zookeeper/filesystem/FileSystemStateStorageHelper.java
index d6b69e4..bb7d94c 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/zookeeper/filesystem/FileSystemStateStorageHelper.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/zookeeper/filesystem/FileSystemStateStorageHelper.java
@@ -24,7 +24,7 @@ import org.apache.flink.core.fs.FileSystem;
import org.apache.flink.core.fs.Path;
import org.apache.flink.runtime.state.StateHandle;
import org.apache.flink.runtime.state.filesystem.FileSerializableStateHandle;
-import org.apache.flink.runtime.util.FileUtils;
+import org.apache.flink.util.FileUtils;
import org.apache.flink.runtime.zookeeper.StateStorageHelper;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
index 6d765b3..49953a6 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
@@ -67,7 +67,7 @@ import org.apache.flink.runtime.security.SecurityUtils
import org.apache.flink.runtime.security.SecurityUtils.FlinkSecuredRunner
import org.apache.flink.runtime.util._
import org.apache.flink.runtime.{FlinkActor, LeaderSessionMessageFilter, LogMessages}
-import org.apache.flink.util.NetUtils
+import org.apache.flink.util.{MathUtils, NetUtils}
import scala.collection.JavaConversions._
import scala.collection.JavaConverters._
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/CommonTestUtils.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/CommonTestUtils.java b/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/CommonTestUtils.java
index bbb6a89..6bd8b34 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/CommonTestUtils.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/CommonTestUtils.java
@@ -18,7 +18,7 @@
package org.apache.flink.runtime.testutils;
-import org.apache.flink.runtime.util.FileUtils;
+import org.apache.flink.util.FileUtils;
import java.io.File;
import java.io.FileWriter;
[04/12] flink git commit: [FLINK-2544] [docs] Add Java 8 version for
building PowerMock tests to docs
Posted by fh...@apache.org.
[FLINK-2544] [docs] Add Java 8 version for building PowerMock tests to docs
Java 8 update 11 introduced a stricter bytecode verifier that leads
to failures in unit tests that use the PowerMock runner. The tests
run correctly in Java 8u51 or above.
This closes #1882
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/d938c5f5
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/d938c5f5
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/d938c5f5
Branch: refs/heads/master
Commit: d938c5f592addcee75227502a930264b36306aed
Parents: dc78a74
Author: spkavuly <so...@intel.com>
Authored: Wed Apr 13 12:02:57 2016 -0700
Committer: Fabian Hueske <fh...@apache.org>
Committed: Fri Apr 15 19:38:15 2016 +0200
----------------------------------------------------------------------
README.md | 3 ++-
docs/setup/building.md | 3 +++
2 files changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/d938c5f5/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index 1597a58..8fd533b 100644
--- a/README.md
+++ b/README.md
@@ -80,7 +80,8 @@ mvn clean package -DskipTests # this will take up to 10 minutes
Flink is now installed in `build-target`
-*NOTE: Maven 3.3.x can build Flink, but will not properly shade away certain dependencies. Maven 3.0.3 creates the libraries properly.*
+*NOTE: Maven 3.3.x can build Flink, but will not properly shade away certain dependencies. Maven 3.0.3 creates the libraries properly.
+To build unit tests with Java 8, use Java 8u51 or above to prevent failures in unit tests that use the PowerMock runner.*
## Developing Flink
http://git-wip-us.apache.org/repos/asf/flink/blob/d938c5f5/docs/setup/building.md
----------------------------------------------------------------------
diff --git a/docs/setup/building.md b/docs/setup/building.md
index a1e768b..1848466 100644
--- a/docs/setup/building.md
+++ b/docs/setup/building.md
@@ -34,6 +34,9 @@ In order to build Flink you need the source code. Either [download the source of
In addition you need **Maven 3** and a **JDK** (Java Development Kit). Flink requires **at least Java 7** to build. We recommend using Java 8.
+*NOTE: Maven 3.3.x can build Flink, but will not properly shade away certain dependencies. Maven 3.0.3 creates the libraries properly.
+To build unit tests with Java 8, use Java 8u51 or above to prevent failures in unit tests that use the PowerMock runner.*
+
To clone from git, enter:
~~~bash
[07/12] flink git commit: [FLINK-3738] [table] Refactor
TableEnvironments. Remove Translators and TranslationContext.
Posted by fh...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/TableEnvironmentITCase.java
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/TableEnvironmentITCase.java b/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/TableEnvironmentITCase.java
new file mode 100644
index 0000000..c596014
--- /dev/null
+++ b/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/TableEnvironmentITCase.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.java.table.test;
+
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.table.BatchTableEnvironment;
+import org.apache.flink.api.java.tuple.Tuple3;
+import org.apache.flink.api.java.tuple.Tuple5;
+import org.apache.flink.api.table.Row;
+import org.apache.flink.api.table.Table;
+import org.apache.flink.api.table.TableEnvironment;
+import org.apache.flink.api.table.TableException;
+import org.apache.flink.api.table.test.utils.TableProgramsTestBase;
+import org.apache.flink.test.javaApiOperators.util.CollectionDataSets;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.util.List;
+
+@RunWith(Parameterized.class)
+public class TableEnvironmentITCase extends TableProgramsTestBase {
+
+ public TableEnvironmentITCase(TestExecutionMode mode, TableConfigMode configMode) {
+ super(mode, configMode);
+ }
+
+ @Test
+ public void testSimpleRegister() throws Exception {
+ final String tableName = "MyTable";
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env, config());
+
+ DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
+ tableEnv.registerDataSet(tableName, ds);
+ Table t = tableEnv.scan(tableName);
+
+ Table result = t.select("f0, f1");
+
+ DataSet<Row> resultSet = tableEnv.toDataSet(result, Row.class);
+ List<Row> results = resultSet.collect();
+ String expected = "1,1\n" + "2,2\n" + "3,2\n" + "4,3\n" + "5,3\n" + "6,3\n" + "7,4\n" +
+ "8,4\n" + "9,4\n" + "10,4\n" + "11,5\n" + "12,5\n" + "13,5\n" + "14,5\n" + "15,5\n" +
+ "16,6\n" + "17,6\n" + "18,6\n" + "19,6\n" + "20,6\n" + "21,6\n";
+ compareResultAsText(results, expected);
+ }
+
+ @Test
+ public void testRegisterWithFields() throws Exception {
+ final String tableName = "MyTable";
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env, config());
+
+ DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
+ tableEnv.registerDataSet(tableName, ds, "a, b, c");
+ Table t = tableEnv.scan(tableName);
+
+ Table result = t.select("a, b, c");
+
+ DataSet<Row> resultSet = tableEnv.toDataSet(result, Row.class);
+ List<Row> results = resultSet.collect();
+ String expected = "1,1,Hi\n" + "2,2,Hello\n" + "3,2,Hello world\n" +
+ "4,3,Hello world, how are you?\n" + "5,3,I am fine.\n" + "6,3,Luke Skywalker\n" +
+ "7,4,Comment#1\n" + "8,4,Comment#2\n" + "9,4,Comment#3\n" + "10,4,Comment#4\n" +
+ "11,5,Comment#5\n" + "12,5,Comment#6\n" + "13,5,Comment#7\n" +
+ "14,5,Comment#8\n" + "15,5,Comment#9\n" + "16,6,Comment#10\n" +
+ "17,6,Comment#11\n" + "18,6,Comment#12\n" + "19,6,Comment#13\n" +
+ "20,6,Comment#14\n" + "21,6,Comment#15\n";
+ compareResultAsText(results, expected);
+ }
+
+ @Test(expected = TableException.class)
+ public void testRegisterExistingDatasetTable() throws Exception {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env, config());
+
+ DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
+ tableEnv.registerDataSet("MyTable", ds);
+ DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 =
+ CollectionDataSets.getSmall5TupleDataSet(env);
+ // Must fail. Name is already used for different table.
+ tableEnv.registerDataSet("MyTable", ds2);
+ }
+
+ @Test(expected = TableException.class)
+ public void testScanUnregisteredTable() throws Exception {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env, config());
+
+ // Must fail. No table registered under that name.
+ tableEnv.scan("nonRegisteredTable");
+ }
+
+ @Test
+ public void testTableRegister() throws Exception {
+ final String tableName = "MyTable";
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env, config());
+
+ DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
+ Table t = tableEnv.fromDataSet(ds);
+ tableEnv.registerTable(tableName, t);
+ Table result = tableEnv.scan(tableName).select("f0, f1").filter("f0 > 7");
+
+ DataSet<Row> resultSet = tableEnv.toDataSet(result, Row.class);
+ List<Row> results = resultSet.collect();
+ String expected = "8,4\n" + "9,4\n" + "10,4\n" + "11,5\n" + "12,5\n" +
+ "13,5\n" + "14,5\n" + "15,5\n" +
+ "16,6\n" + "17,6\n" + "18,6\n" + "19,6\n" + "20,6\n" + "21,6\n";
+ compareResultAsText(results, expected);
+ }
+
+ @Test(expected = TableException.class)
+ public void testIllegalName() throws Exception {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env, config());
+
+ DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
+ Table t = tableEnv.fromDataSet(ds);
+ // Must fail. Table name matches internal name pattern.
+ tableEnv.registerTable("_DataSetTable_42", t);
+ }
+
+ @Test(expected = TableException.class)
+ public void testRegisterTableFromOtherEnv() throws Exception {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ BatchTableEnvironment tableEnv1 = TableEnvironment.getTableEnvironment(env, config());
+ BatchTableEnvironment tableEnv2 = TableEnvironment.getTableEnvironment(env, config());
+
+ Table t = tableEnv1.fromDataSet(CollectionDataSets.get3TupleDataSet(env));
+ // Must fail. Table is bound to different TableEnvironment.
+ tableEnv2.registerTable("MyTable", t);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/UnionITCase.java
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/UnionITCase.java b/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/UnionITCase.java
index 167e45c..b69fec4 100644
--- a/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/UnionITCase.java
+++ b/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/UnionITCase.java
@@ -20,11 +20,13 @@ package org.apache.flink.api.java.table.test;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.table.TableEnvironment;
+import org.apache.flink.api.java.table.BatchTableEnvironment;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.api.java.tuple.Tuple5;
import org.apache.flink.api.table.Row;
import org.apache.flink.api.table.Table;
+import org.apache.flink.api.table.TableEnvironment;
+import org.apache.flink.api.table.TableException;
import org.apache.flink.test.javaApiOperators.util.CollectionDataSets;
import org.apache.flink.test.util.MultipleProgramsTestBase;
import org.junit.Test;
@@ -43,7 +45,7 @@ public class UnionITCase extends MultipleProgramsTestBase {
@Test
public void testUnion() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = new TableEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
DataSet<Tuple3<Integer, Long, String>> ds1 = CollectionDataSets.getSmall3TupleDataSet(env);
DataSet<Tuple3<Integer, Long, String>> ds2 = CollectionDataSets.getSmall3TupleDataSet(env);
@@ -62,7 +64,7 @@ public class UnionITCase extends MultipleProgramsTestBase {
@Test
public void testUnionWithFilter() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = new TableEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
DataSet<Tuple3<Integer, Long, String>> ds1 = CollectionDataSets.getSmall3TupleDataSet(env);
DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.get5TupleDataSet(env);
@@ -81,7 +83,7 @@ public class UnionITCase extends MultipleProgramsTestBase {
@Test(expected = IllegalArgumentException.class)
public void testUnionIncompatibleNumberOfFields() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = new TableEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
DataSet<Tuple3<Integer, Long, String>> ds1 = CollectionDataSets.getSmall3TupleDataSet(env);
DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.get5TupleDataSet(env);
@@ -96,7 +98,7 @@ public class UnionITCase extends MultipleProgramsTestBase {
@Test(expected = IllegalArgumentException.class)
public void testUnionIncompatibleFieldsName() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = new TableEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
DataSet<Tuple3<Integer, Long, String>> ds1 = CollectionDataSets.getSmall3TupleDataSet(env);
DataSet<Tuple3<Integer, Long, String>> ds2 = CollectionDataSets.getSmall3TupleDataSet(env);
@@ -111,7 +113,7 @@ public class UnionITCase extends MultipleProgramsTestBase {
@Test(expected = IllegalArgumentException.class)
public void testUnionIncompatibleFieldTypes() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = new TableEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
DataSet<Tuple3<Integer, Long, String>> ds1 = CollectionDataSets.getSmall3TupleDataSet(env);
DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.get5TupleDataSet(env);
@@ -126,7 +128,7 @@ public class UnionITCase extends MultipleProgramsTestBase {
@Test
public void testUnionWithAggregation() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = new TableEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
DataSet<Tuple3<Integer, Long, String>> ds1 = CollectionDataSets.getSmall3TupleDataSet(env);
DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.get5TupleDataSet(env);
@@ -145,7 +147,7 @@ public class UnionITCase extends MultipleProgramsTestBase {
@Test
public void testUnionWithJoin() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = new TableEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
DataSet<Tuple3<Integer, Long, String>> ds1 = CollectionDataSets.getSmall3TupleDataSet(env);
DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.get5TupleDataSet(env);
@@ -164,5 +166,21 @@ public class UnionITCase extends MultipleProgramsTestBase {
"Hallo Welt,Hallo Welt\n" + "Hallo Welt wie,Hallo Welt\n" +
"Hallo Welt,Hallo Welt wie\n" + "Hallo Welt wie,Hallo Welt wie\n";
compareResultAsText(results, expected);
- }
+ }
+
+ @Test(expected = TableException.class)
+ public void testUnionTablesFromDifferentEnvs() throws Exception {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ BatchTableEnvironment tEnv1 = TableEnvironment.getTableEnvironment(env);
+ BatchTableEnvironment tEnv2 = TableEnvironment.getTableEnvironment(env);
+
+ DataSet<Tuple3<Integer, Long, String>> ds1 = CollectionDataSets.getSmall3TupleDataSet(env);
+ DataSet<Tuple3<Integer, Long, String>> ds2 = CollectionDataSets.getSmall3TupleDataSet(env);
+
+ Table in1 = tEnv1.fromDataSet(ds1, "a, b, c");
+ Table in2 = tEnv2.fromDataSet(ds2, "a, b, c");
+
+ // Must fail. Tables are bound to different TableEnvironments.
+ in1.unionAll(in2);
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/sql/test/AggregationsITCase.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/sql/test/AggregationsITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/sql/test/AggregationsITCase.scala
index d577564..08f44d3 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/sql/test/AggregationsITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/sql/test/AggregationsITCase.scala
@@ -21,8 +21,7 @@ package org.apache.flink.api.scala.sql.test
import org.apache.flink.api.scala._
import org.apache.flink.api.scala.table._
import org.apache.flink.api.scala.util.CollectionDataSets
-import org.apache.flink.api.table.Row
-import org.apache.flink.api.table.plan.TranslationContext
+import org.apache.flink.api.table.{TableEnvironment, Row}
import org.apache.flink.api.table.test.utils.TableProgramsTestBase
import org.apache.flink.api.table.test.utils.TableProgramsTestBase.TableConfigMode
import org.apache.flink.test.util.MultipleProgramsTestBase.TestExecutionMode
@@ -43,8 +42,7 @@ class AggregationsITCase(
def testAggregationTypes(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val tEnv = getScalaTableEnvironment
- TranslationContext.reset()
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
val sqlQuery = "SELECT sum(_1), min(_1), max(_1), count(_1), avg(_1) FROM MyTable"
@@ -54,7 +52,7 @@ class AggregationsITCase(
val result = tEnv.sql(sqlQuery)
val expected = "231,1,21,21,11"
- val results = result.toDataSet[Row](getConfig).collect()
+ val results = result.toDataSet[Row].collect()
TestBaseUtils.compareResultAsText(results.asJava, expected)
}
@@ -62,18 +60,17 @@ class AggregationsITCase(
def testTableAggregation(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val tEnv = getScalaTableEnvironment
- TranslationContext.reset()
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
val sqlQuery = "SELECT sum(_1) FROM MyTable"
- val ds = CollectionDataSets.get3TupleDataSet(env).toTable
+ val ds = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv)
tEnv.registerTable("MyTable", ds)
val result = tEnv.sql(sqlQuery)
val expected = "231"
- val results = result.toDataSet[Row](getConfig).collect()
+ val results = result.toDataSet[Row].collect()
TestBaseUtils.compareResultAsText(results.asJava, expected)
}
@@ -81,8 +78,7 @@ class AggregationsITCase(
def testDataSetAggregation(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val tEnv = getScalaTableEnvironment
- TranslationContext.reset()
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
val sqlQuery = "SELECT sum(_1) FROM MyTable"
@@ -92,7 +88,7 @@ class AggregationsITCase(
val result = tEnv.sql(sqlQuery)
val expected = "231"
- val results = result.toDataSet[Row](getConfig).collect()
+ val results = result.toDataSet[Row].collect()
TestBaseUtils.compareResultAsText(results.asJava, expected)
}
@@ -100,8 +96,7 @@ class AggregationsITCase(
def testWorkingAggregationDataTypes(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val tEnv = getScalaTableEnvironment
- TranslationContext.reset()
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
val sqlQuery = "SELECT avg(_1), avg(_2), avg(_3), avg(_4), avg(_5), avg(_6), count(_7)" +
"FROM MyTable"
@@ -114,7 +109,7 @@ class AggregationsITCase(
val result = tEnv.sql(sqlQuery)
val expected = "1,1,1,1,1.5,1.5,2"
- val results = result.toDataSet[Row](getConfig).collect()
+ val results = result.toDataSet[Row].collect()
TestBaseUtils.compareResultAsText(results.asJava, expected)
}
@@ -122,21 +117,20 @@ class AggregationsITCase(
def testTableWorkingAggregationDataTypes(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val tEnv = getScalaTableEnvironment
- TranslationContext.reset()
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
val sqlQuery = "SELECT avg(a), avg(b), avg(c), avg(d), avg(e), avg(f), count(g)" +
"FROM MyTable"
val ds = env.fromElements(
(1: Byte, 1: Short, 1, 1L, 1.0f, 1.0d, "Hello"),
- (2: Byte, 2: Short, 2, 2L, 2.0f, 2.0d, "Ciao")).as('a, 'b, 'c, 'd, 'e, 'f, 'g)
+ (2: Byte, 2: Short, 2, 2L, 2.0f, 2.0d, "Ciao")).toTable(tEnv, 'a, 'b, 'c, 'd, 'e, 'f, 'g)
tEnv.registerTable("MyTable", ds)
val result = tEnv.sql(sqlQuery)
val expected = "1,1,1,1,1.5,1.5,2"
- val results = result.toDataSet[Row](getConfig).collect()
+ val results = result.toDataSet[Row].collect()
TestBaseUtils.compareResultAsText(results.asJava, expected)
}
@@ -144,19 +138,18 @@ class AggregationsITCase(
def testTableProjection(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val tEnv = getScalaTableEnvironment
- TranslationContext.reset()
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
val sqlQuery = "SELECT avg(a), sum(a), count(a), avg(b), sum(b) " +
"FROM MyTable"
- val ds = env.fromElements((1: Byte, 1: Short), (2: Byte, 2: Short)).as('a, 'b)
+ val ds = env.fromElements((1: Byte, 1: Short), (2: Byte, 2: Short)).toTable(tEnv, 'a, 'b)
tEnv.registerTable("MyTable", ds)
val result = tEnv.sql(sqlQuery)
val expected = "1,3,2,1,3"
- val results = result.toDataSet[Row](getConfig).collect()
+ val results = result.toDataSet[Row].collect()
TestBaseUtils.compareResultAsText(results.asJava, expected)
}
@@ -164,19 +157,18 @@ class AggregationsITCase(
def testTableAggregationWithArithmetic(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val tEnv = getScalaTableEnvironment
- TranslationContext.reset()
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
val sqlQuery = "SELECT avg(a + 2) + 2, count(b) + 5 " +
"FROM MyTable"
- val ds = env.fromElements((1f, "Hello"), (2f, "Ciao")).as('a, 'b)
+ val ds = env.fromElements((1f, "Hello"), (2f, "Ciao")).toTable(tEnv, 'a, 'b)
tEnv.registerTable("MyTable", ds)
val result = tEnv.sql(sqlQuery)
val expected = "5.5,7"
- val results = result.toDataSet[Row](getConfig).collect()
+ val results = result.toDataSet[Row].collect()
TestBaseUtils.compareResultAsText(results.asJava, expected)
}
@@ -184,18 +176,17 @@ class AggregationsITCase(
def testAggregationWithTwoCount(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val tEnv = getScalaTableEnvironment
- TranslationContext.reset()
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
val sqlQuery = "SELECT count(_1), count(_2) FROM MyTable"
- val ds = env.fromElements((1f, "Hello"), (2f, "Ciao")).toTable
+ val ds = env.fromElements((1f, "Hello"), (2f, "Ciao")).toTable(tEnv)
tEnv.registerTable("MyTable", ds)
val result = tEnv.sql(sqlQuery)
val expected = "2,2"
- val results = result.toDataSet[Row](getConfig).collect()
+ val results = result.toDataSet[Row].collect()
TestBaseUtils.compareResultAsText(results.asJava, expected)
}
@@ -204,21 +195,20 @@ class AggregationsITCase(
def testAggregationAfterProjection(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val tEnv = getScalaTableEnvironment
- TranslationContext.reset()
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
val sqlQuery = "SELECT avg(a), sum(b), count(c) FROM " +
"(SELECT _1 as a, _2 as b, _3 as c FROM MyTable)"
val ds = env.fromElements(
(1: Byte, 1: Short, 1, 1L, 1.0f, 1.0d, "Hello"),
- (2: Byte, 2: Short, 2, 2L, 2.0f, 2.0d, "Ciao")).toTable
+ (2: Byte, 2: Short, 2, 2L, 2.0f, 2.0d, "Ciao")).toTable(tEnv)
tEnv.registerTable("MyTable", ds)
val result = tEnv.sql(sqlQuery)
val expected = "1,3,2"
- val results = result.toDataSet[Row](getConfig).collect()
+ val results = result.toDataSet[Row].collect()
TestBaseUtils.compareResultAsText(results.asJava, expected)
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/sql/test/FilterITCase.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/sql/test/FilterITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/sql/test/FilterITCase.scala
index 171e200..e1edd87 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/sql/test/FilterITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/sql/test/FilterITCase.scala
@@ -21,8 +21,7 @@ package org.apache.flink.api.scala.sql.test
import org.apache.flink.api.scala._
import org.apache.flink.api.scala.table._
import org.apache.flink.api.scala.util.CollectionDataSets
-import org.apache.flink.api.table.Row
-import org.apache.flink.api.table.plan.TranslationContext
+import org.apache.flink.api.table.{TableEnvironment, Row}
import org.apache.flink.api.table.test.utils.TableProgramsTestBase
import org.apache.flink.api.table.test.utils.TableProgramsTestBase.TableConfigMode
import org.apache.flink.test.util.MultipleProgramsTestBase.TestExecutionMode
@@ -43,18 +42,17 @@ class FilterITCase(
def testAllRejectingFilter(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val tEnv = getScalaTableEnvironment
- TranslationContext.reset()
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
val sqlQuery = "SELECT * FROM MyTable WHERE false"
- val ds = CollectionDataSets.get3TupleDataSet(env).toTable
+ val ds = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv)
tEnv.registerTable("MyTable", ds)
val result = tEnv.sql(sqlQuery)
val expected = "\n"
- val results = result.toDataSet[Row](getConfig).collect()
+ val results = result.toDataSet[Row].collect()
TestBaseUtils.compareResultAsText(results.asJava, expected)
}
@@ -62,12 +60,11 @@ class FilterITCase(
def testAllPassingFilter(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val tEnv = getScalaTableEnvironment
- TranslationContext.reset()
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
val sqlQuery = "SELECT * FROM MyTable WHERE true"
- val ds = CollectionDataSets.get3TupleDataSet(env).toTable
+ val ds = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv)
tEnv.registerTable("MyTable", ds)
val result = tEnv.sql(sqlQuery)
@@ -78,7 +75,7 @@ class FilterITCase(
"Comment#5\n" + "12,5,Comment#6\n" + "13,5,Comment#7\n" + "14,5,Comment#8\n" + "15,5," +
"Comment#9\n" + "16,6,Comment#10\n" + "17,6,Comment#11\n" + "18,6,Comment#12\n" + "19," +
"6,Comment#13\n" + "20,6,Comment#14\n" + "21,6,Comment#15\n"
- val results = result.toDataSet[Row](getConfig).collect()
+ val results = result.toDataSet[Row].collect()
TestBaseUtils.compareResultAsText(results.asJava, expected)
}
@@ -86,18 +83,17 @@ class FilterITCase(
def testFilterOnString(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val tEnv = getScalaTableEnvironment
- TranslationContext.reset()
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
val sqlQuery = "SELECT * FROM MyTable WHERE c LIKE '%world%'"
- val ds = CollectionDataSets.get3TupleDataSet(env).toTable.as('a, 'b, 'c)
+ val ds = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv).as('a, 'b, 'c)
tEnv.registerTable("MyTable", ds)
val result = tEnv.sql(sqlQuery)
val expected = "3,2,Hello world\n" + "4,3,Hello world, how are you?\n"
- val results = result.toDataSet[Row](getConfig).collect()
+ val results = result.toDataSet[Row].collect()
TestBaseUtils.compareResultAsText(results.asJava, expected)
}
@@ -105,12 +101,11 @@ class FilterITCase(
def testFilterOnInteger(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val tEnv = getScalaTableEnvironment
- TranslationContext.reset()
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
val sqlQuery = "SELECT * FROM MyTable WHERE MOD(a,2)=0"
- val ds = CollectionDataSets.get3TupleDataSet(env).toTable.as('a, 'b, 'c)
+ val ds = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv).as('a, 'b, 'c)
tEnv.registerTable("MyTable", ds)
val result = tEnv.sql(sqlQuery)
@@ -119,7 +114,7 @@ class FilterITCase(
"6,3,Luke Skywalker\n" + "8,4," + "Comment#2\n" + "10,4,Comment#4\n" +
"12,5,Comment#6\n" + "14,5,Comment#8\n" + "16,6," +
"Comment#10\n" + "18,6,Comment#12\n" + "20,6,Comment#14\n"
- val results = result.toDataSet[Row](getConfig).collect()
+ val results = result.toDataSet[Row].collect()
TestBaseUtils.compareResultAsText(results.asJava, expected)
}
@@ -127,18 +122,17 @@ class FilterITCase(
def testDisjunctivePredicate(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val tEnv = getScalaTableEnvironment
- TranslationContext.reset()
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
val sqlQuery = "SELECT * FROM MyTable WHERE a < 2 OR a > 20"
- val ds = CollectionDataSets.get3TupleDataSet(env).toTable.as('a, 'b, 'c)
+ val ds = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv).as('a, 'b, 'c)
tEnv.registerTable("MyTable", ds)
val result = tEnv.sql(sqlQuery)
val expected = "1,1,Hi\n" + "21,6,Comment#15\n"
- val results = result.toDataSet[Row](getConfig).collect()
+ val results = result.toDataSet[Row].collect()
TestBaseUtils.compareResultAsText(results.asJava, expected)
}
@@ -146,12 +140,11 @@ class FilterITCase(
def testFilterWithAnd(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val tEnv = getScalaTableEnvironment
- TranslationContext.reset()
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
val sqlQuery = "SELECT * FROM MyTable WHERE MOD(a,2)<>0 AND MOD(b,2)=0"
- val ds = CollectionDataSets.get3TupleDataSet(env).toTable.as('a, 'b, 'c)
+ val ds = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv).as('a, 'b, 'c)
tEnv.registerTable("MyTable", ds)
val result = tEnv.sql(sqlQuery)
@@ -159,7 +152,7 @@ class FilterITCase(
val expected = "3,2,Hello world\n" + "7,4,Comment#1\n" +
"9,4,Comment#3\n" + "17,6,Comment#11\n" +
"19,6,Comment#13\n" + "21,6,Comment#15\n"
- val results = result.toDataSet[Row](getConfig).collect()
+ val results = result.toDataSet[Row].collect()
TestBaseUtils.compareResultAsText(results.asJava, expected)
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/sql/test/JoinITCase.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/sql/test/JoinITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/sql/test/JoinITCase.scala
index b09aa75..561f2a7 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/sql/test/JoinITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/sql/test/JoinITCase.scala
@@ -22,8 +22,8 @@ import org.apache.calcite.tools.ValidationException
import org.apache.flink.api.scala._
import org.apache.flink.api.scala.table._
import org.apache.flink.api.scala.util.CollectionDataSets
-import org.apache.flink.api.table.{TableException, Row}
-import org.apache.flink.api.table.plan.{PlanGenException, TranslationContext}
+import org.apache.flink.api.table.{TableEnvironment, TableException, Row}
+import org.apache.flink.api.table.plan.PlanGenException
import org.apache.flink.api.table.test.utils.TableProgramsTestBase
import org.apache.flink.api.table.test.utils.TableProgramsTestBase.TableConfigMode
import org.apache.flink.test.util.MultipleProgramsTestBase.TestExecutionMode
@@ -44,20 +44,19 @@ class JoinITCase(
def testJoin(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val tEnv = getScalaTableEnvironment
- TranslationContext.reset()
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
val sqlQuery = "SELECT c, g FROM Table3, Table5 WHERE b = e"
- val ds1 = CollectionDataSets.getSmall3TupleDataSet(env).toTable.as('a, 'b, 'c)
- val ds2 = CollectionDataSets.get5TupleDataSet(env).toTable.as('d, 'e, 'f, 'g, 'h)
+ val ds1 = CollectionDataSets.getSmall3TupleDataSet(env).toTable(tEnv).as('a, 'b, 'c)
+ val ds2 = CollectionDataSets.get5TupleDataSet(env).toTable(tEnv).as('d, 'e, 'f, 'g, 'h)
tEnv.registerTable("Table3", ds1)
tEnv.registerTable("Table5", ds2)
val result = tEnv.sql(sqlQuery)
val expected = "Hi,Hallo\n" + "Hello,Hallo Welt\n" + "Hello world,Hallo Welt\n"
- val results = result.toDataSet[Row](getConfig).collect()
+ val results = result.toDataSet[Row].collect()
TestBaseUtils.compareResultAsText(results.asJava, expected)
}
@@ -65,20 +64,19 @@ class JoinITCase(
def testJoinWithFilter(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val tEnv = getScalaTableEnvironment
- TranslationContext.reset()
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
val sqlQuery = "SELECT c, g FROM Table3, Table5 WHERE b = e AND b < 2"
- val ds1 = CollectionDataSets.getSmall3TupleDataSet(env).toTable.as('a, 'b, 'c)
- val ds2 = CollectionDataSets.get5TupleDataSet(env).toTable.as('d, 'e, 'f, 'g, 'h)
+ val ds1 = CollectionDataSets.getSmall3TupleDataSet(env).toTable(tEnv).as('a, 'b, 'c)
+ val ds2 = CollectionDataSets.get5TupleDataSet(env).toTable(tEnv).as('d, 'e, 'f, 'g, 'h)
tEnv.registerTable("Table3", ds1)
tEnv.registerTable("Table5", ds2)
val result = tEnv.sql(sqlQuery)
val expected = "Hi,Hallo\n"
- val results = result.toDataSet[Row](getConfig).collect()
+ val results = result.toDataSet[Row].collect()
TestBaseUtils.compareResultAsText(results.asJava, expected)
}
@@ -86,13 +84,12 @@ class JoinITCase(
def testJoinWithJoinFilter(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val tEnv = getScalaTableEnvironment
- TranslationContext.reset()
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
val sqlQuery = "SELECT c, g FROM Table3, Table5 WHERE b = e AND a < 6 AND h < b"
- val ds1 = CollectionDataSets.get3TupleDataSet(env).toTable.as('a, 'b, 'c)
- val ds2 = CollectionDataSets.get5TupleDataSet(env).toTable.as('d, 'e, 'f, 'g, 'h)
+ val ds1 = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv).as('a, 'b, 'c)
+ val ds2 = CollectionDataSets.get5TupleDataSet(env).toTable(tEnv).as('d, 'e, 'f, 'g, 'h)
tEnv.registerTable("Table3", ds1)
tEnv.registerTable("Table5", ds2)
@@ -100,7 +97,7 @@ class JoinITCase(
val expected = "Hello world, how are you?,Hallo Welt wie\n" +
"I am fine.,Hallo Welt wie\n"
- val results = result.toDataSet[Row](getConfig).collect()
+ val results = result.toDataSet[Row].collect()
TestBaseUtils.compareResultAsText(results.asJava, expected)
}
@@ -108,13 +105,12 @@ class JoinITCase(
def testJoinWithMultipleKeys(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val tEnv = getScalaTableEnvironment
- TranslationContext.reset()
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
val sqlQuery = "SELECT c, g FROM Table3, Table5 WHERE a = d AND b = h"
- val ds1 = CollectionDataSets.get3TupleDataSet(env).toTable.as('a, 'b, 'c)
- val ds2 = CollectionDataSets.get5TupleDataSet(env).toTable.as('d, 'e, 'f, 'g, 'h)
+ val ds1 = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv).as('a, 'b, 'c)
+ val ds2 = CollectionDataSets.get5TupleDataSet(env).toTable(tEnv).as('d, 'e, 'f, 'g, 'h)
tEnv.registerTable("Table3", ds1)
tEnv.registerTable("Table5", ds2)
@@ -122,7 +118,7 @@ class JoinITCase(
val expected = "Hi,Hallo\n" + "Hello,Hallo Welt\n" + "Hello world,Hallo Welt wie gehts?\n" +
"Hello world,ABC\n" + "I am fine.,HIJ\n" + "I am fine.,IJK\n"
- val results = result.toDataSet[Row](getConfig).collect()
+ val results = result.toDataSet[Row].collect()
TestBaseUtils.compareResultAsText(results.asJava, expected)
}
@@ -130,13 +126,12 @@ class JoinITCase(
def testJoinNonExistingKey(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val tEnv = getScalaTableEnvironment
- TranslationContext.reset()
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
val sqlQuery = "SELECT c, g FROM Table3, Table5 WHERE foo = e"
- val ds1 = CollectionDataSets.get3TupleDataSet(env).toTable.as('a, 'b, 'c)
- val ds2 = CollectionDataSets.get5TupleDataSet(env).toTable.as('d, 'e, 'f, 'g, 'h)
+ val ds1 = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv).as('a, 'b, 'c)
+ val ds2 = CollectionDataSets.get5TupleDataSet(env).toTable(tEnv).as('d, 'e, 'f, 'g, 'h)
tEnv.registerTable("Table3", ds1)
tEnv.registerTable("Table5", ds2)
@@ -147,13 +142,12 @@ class JoinITCase(
def testJoinNonMatchingKeyTypes(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val tEnv = getScalaTableEnvironment
- TranslationContext.reset()
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
val sqlQuery = "SELECT c, g FROM Table3, Table5 WHERE a = g"
- val ds1 = CollectionDataSets.get3TupleDataSet(env).toTable.as('a, 'b, 'c)
- val ds2 = CollectionDataSets.get5TupleDataSet(env).toTable.as('d, 'e, 'f, 'g, 'h)
+ val ds1 = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv).as('a, 'b, 'c)
+ val ds2 = CollectionDataSets.get5TupleDataSet(env).toTable(tEnv).as('d, 'e, 'f, 'g, 'h)
tEnv.registerTable("Table3", ds1)
tEnv.registerTable("Table5", ds2)
@@ -164,13 +158,12 @@ class JoinITCase(
def testJoinWithAmbiguousFields(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val tEnv = getScalaTableEnvironment
- TranslationContext.reset()
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
val sqlQuery = "SELECT c, g FROM Table3, Table5 WHERE a = d"
- val ds1 = CollectionDataSets.get3TupleDataSet(env).toTable.as('a, 'b, 'c)
- val ds2 = CollectionDataSets.get5TupleDataSet(env).toTable.as('d, 'e, 'f, 'g, 'c)
+ val ds1 = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv).as('a, 'b, 'c)
+ val ds2 = CollectionDataSets.get5TupleDataSet(env).toTable(tEnv).as('d, 'e, 'f, 'g, 'c)
tEnv.registerTable("Table3", ds1)
tEnv.registerTable("Table5", ds2)
@@ -181,20 +174,19 @@ class JoinITCase(
def testJoinWithAlias(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val tEnv = getScalaTableEnvironment
- TranslationContext.reset()
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
val sqlQuery = "SELECT Table5.c, Table3.c FROM Table3, Table5 WHERE a = d AND a < 4"
- val ds1 = CollectionDataSets.get3TupleDataSet(env).toTable.as('a, 'b, 'c)
- val ds2 = CollectionDataSets.get5TupleDataSet(env).toTable.as('d, 'e, 'f, 'g, 'c)
+ val ds1 = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv).as('a, 'b, 'c)
+ val ds2 = CollectionDataSets.get5TupleDataSet(env).toTable(tEnv).as('d, 'e, 'f, 'g, 'c)
tEnv.registerTable("Table3", ds1)
tEnv.registerTable("Table5", ds2)
val result = tEnv.sql(sqlQuery)
val expected = "1,Hi\n" + "2,Hello\n" + "1,Hello\n" +
"2,Hello world\n" + "2,Hello world\n" + "3,Hello world\n"
- val results = result.toDataSet[Row](getConfig).collect()
+ val results = result.toDataSet[Row].collect()
TestBaseUtils.compareResultAsText(results.asJava, expected)
}
@@ -202,25 +194,23 @@ class JoinITCase(
def testJoinNoEqualityPredicate(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val tEnv = getScalaTableEnvironment
- TranslationContext.reset()
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
val sqlQuery = "SELECT c, g FROM Table3, Table5 WHERE d = f"
- val ds1 = CollectionDataSets.get3TupleDataSet(env).toTable.as('a, 'b, 'c)
- val ds2 = CollectionDataSets.get5TupleDataSet(env).toTable.as('d, 'e, 'f, 'g, 'h)
+ val ds1 = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv).as('a, 'b, 'c)
+ val ds2 = CollectionDataSets.get5TupleDataSet(env).toTable(tEnv).as('d, 'e, 'f, 'g, 'h)
tEnv.registerTable("Table3", ds1)
tEnv.registerTable("Table5", ds2)
- tEnv.sql(sqlQuery).toDataSet[Row](getConfig).collect()
+ tEnv.sql(sqlQuery).toDataSet[Row].collect()
}
@Test
def testDataSetJoinWithAggregation(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val tEnv = getScalaTableEnvironment
- TranslationContext.reset()
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
val sqlQuery = "SELECT COUNT(g), COUNT(b) FROM Table3, Table5 WHERE a = d"
@@ -232,7 +222,7 @@ class JoinITCase(
val result = tEnv.sql(sqlQuery)
val expected = "6,6"
- val results = result.toDataSet[Row](getConfig).collect()
+ val results = result.toDataSet[Row].collect()
TestBaseUtils.compareResultAsText(results.asJava, expected)
}
@@ -240,20 +230,19 @@ class JoinITCase(
def testTableJoinWithAggregation(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val tEnv = getScalaTableEnvironment
- TranslationContext.reset()
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
val sqlQuery = "SELECT COUNT(b), COUNT(g) FROM Table3, Table5 WHERE a = d"
- val ds1 = CollectionDataSets.getSmall3TupleDataSet(env).as('a, 'b, 'c)
- val ds2 = CollectionDataSets.get5TupleDataSet(env).as('d, 'e, 'f, 'g, 'h)
+ val ds1 = CollectionDataSets.getSmall3TupleDataSet(env).toTable(tEnv, 'a, 'b, 'c)
+ val ds2 = CollectionDataSets.get5TupleDataSet(env).toTable(tEnv, 'd, 'e, 'f, 'g, 'h)
tEnv.registerTable("Table3", ds1)
tEnv.registerTable("Table5", ds2)
val result = tEnv.sql(sqlQuery)
val expected = "6,6"
- val results = result.toDataSet[Row](getConfig).collect()
+ val results = result.toDataSet[Row].collect()
TestBaseUtils.compareResultAsText(results.asJava, expected)
}
@@ -261,50 +250,47 @@ class JoinITCase(
def testFullOuterJoin(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val tEnv = getScalaTableEnvironment
- TranslationContext.reset()
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
val sqlQuery = "SELECT c, g FROM Table3 FULL OUTER JOIN Table5 ON b = e"
- val ds1 = CollectionDataSets.getSmall3TupleDataSet(env).toTable.as('a, 'b, 'c)
- val ds2 = CollectionDataSets.get5TupleDataSet(env).toTable.as('d, 'e, 'f, 'g, 'h)
+ val ds1 = CollectionDataSets.getSmall3TupleDataSet(env).toTable(tEnv).as('a, 'b, 'c)
+ val ds2 = CollectionDataSets.get5TupleDataSet(env).toTable(tEnv).as('d, 'e, 'f, 'g, 'h)
tEnv.registerTable("Table3", ds1)
tEnv.registerTable("Table5", ds2)
- tEnv.sql(sqlQuery).toDataSet[Row](getConfig).collect()
+ tEnv.sql(sqlQuery).toDataSet[Row].collect()
}
@Test(expected = classOf[PlanGenException])
def testLeftOuterJoin(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val tEnv = getScalaTableEnvironment
- TranslationContext.reset()
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
val sqlQuery = "SELECT c, g FROM Table3 LEFT OUTER JOIN Table5 ON b = e"
- val ds1 = CollectionDataSets.getSmall3TupleDataSet(env).toTable.as('a, 'b, 'c)
- val ds2 = CollectionDataSets.get5TupleDataSet(env).toTable.as('d, 'e, 'f, 'g, 'h)
+ val ds1 = CollectionDataSets.getSmall3TupleDataSet(env).toTable(tEnv).as('a, 'b, 'c)
+ val ds2 = CollectionDataSets.get5TupleDataSet(env).toTable(tEnv).as('d, 'e, 'f, 'g, 'h)
tEnv.registerTable("Table3", ds1)
tEnv.registerTable("Table5", ds2)
- tEnv.sql(sqlQuery).toDataSet[Row](getConfig).collect()
+ tEnv.sql(sqlQuery).toDataSet[Row].collect()
}
@Test(expected = classOf[PlanGenException])
def testRightOuterJoin(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val tEnv = getScalaTableEnvironment
- TranslationContext.reset()
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
val sqlQuery = "SELECT c, g FROM Table3 RIGHT OUTER JOIN Table5 ON b = e"
- val ds1 = CollectionDataSets.getSmall3TupleDataSet(env).toTable.as('a, 'b, 'c)
- val ds2 = CollectionDataSets.get5TupleDataSet(env).toTable.as('d, 'e, 'f, 'g, 'h)
+ val ds1 = CollectionDataSets.getSmall3TupleDataSet(env).toTable(tEnv).as('a, 'b, 'c)
+ val ds2 = CollectionDataSets.get5TupleDataSet(env).toTable(tEnv).as('d, 'e, 'f, 'g, 'h)
tEnv.registerTable("Table3", ds1)
tEnv.registerTable("Table5", ds2)
- tEnv.sql(sqlQuery).toDataSet[Row](getConfig).collect()
+ tEnv.sql(sqlQuery).toDataSet[Row].collect()
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/sql/test/SelectITCase.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/sql/test/SelectITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/sql/test/SelectITCase.scala
index f08c95c..b1d423b 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/sql/test/SelectITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/sql/test/SelectITCase.scala
@@ -22,8 +22,7 @@ import org.apache.calcite.tools.ValidationException
import org.apache.flink.api.scala._
import org.apache.flink.api.scala.table._
import org.apache.flink.api.scala.util.CollectionDataSets
-import org.apache.flink.api.table.Row
-import org.apache.flink.api.table.plan.TranslationContext
+import org.apache.flink.api.table.{TableEnvironment, Row}
import org.apache.flink.api.table.test.utils.TableProgramsTestBase
import org.apache.flink.api.table.test.utils.TableProgramsTestBase.TableConfigMode
import org.apache.flink.test.util.MultipleProgramsTestBase.TestExecutionMode
@@ -44,12 +43,11 @@ class SelectITCase(
def testSelectStarFromTable(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val tEnv = getScalaTableEnvironment
- TranslationContext.reset()
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
val sqlQuery = "SELECT * FROM MyTable"
- val ds = CollectionDataSets.get3TupleDataSet(env).toTable.as('a, 'b, 'c)
+ val ds = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv).as('a, 'b, 'c)
tEnv.registerTable("MyTable", ds)
val result = tEnv.sql(sqlQuery)
@@ -61,7 +59,7 @@ class SelectITCase(
"15,5,Comment#9\n" + "16,6,Comment#10\n" + "17,6,Comment#11\n" + "18,6,Comment#12\n" +
"19,6,Comment#13\n" + "20,6,Comment#14\n" + "21,6,Comment#15\n"
- val results = result.toDataSet[Row](getConfig).collect()
+ val results = result.toDataSet[Row].collect()
TestBaseUtils.compareResultAsText(results.asJava, expected)
}
@@ -69,8 +67,7 @@ class SelectITCase(
def testSelectStarFromDataSet(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val tEnv = getScalaTableEnvironment
- TranslationContext.reset()
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
val sqlQuery = "SELECT * FROM MyTable"
@@ -86,7 +83,7 @@ class SelectITCase(
"15,5,Comment#9\n" + "16,6,Comment#10\n" + "17,6,Comment#11\n" + "18,6,Comment#12\n" +
"19,6,Comment#13\n" + "20,6,Comment#14\n" + "21,6,Comment#15\n"
- val results = result.toDataSet[Row](getConfig).collect()
+ val results = result.toDataSet[Row].collect()
TestBaseUtils.compareResultAsText(results.asJava, expected)
}
@@ -94,12 +91,11 @@ class SelectITCase(
def testSimpleSelectAll(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val tEnv = getScalaTableEnvironment
- TranslationContext.reset()
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
val sqlQuery = "SELECT a, b, c FROM MyTable"
- val ds = CollectionDataSets.get3TupleDataSet(env).toTable.as('a, 'b, 'c)
+ val ds = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv).as('a, 'b, 'c)
tEnv.registerTable("MyTable", ds)
val result = tEnv.sql(sqlQuery)
@@ -111,7 +107,7 @@ class SelectITCase(
"15,5,Comment#9\n" + "16,6,Comment#10\n" + "17,6,Comment#11\n" + "18,6,Comment#12\n" +
"19,6,Comment#13\n" + "20,6,Comment#14\n" + "21,6,Comment#15\n"
- val results = result.toDataSet[Row](getConfig).collect()
+ val results = result.toDataSet[Row].collect()
TestBaseUtils.compareResultAsText(results.asJava, expected)
}
@@ -119,12 +115,11 @@ class SelectITCase(
def testSelectWithNaming(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val tEnv = getScalaTableEnvironment
- TranslationContext.reset()
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
val sqlQuery = "SELECT _1 as a, _2 as b FROM MyTable"
- val ds = CollectionDataSets.get3TupleDataSet(env).toTable
+ val ds = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv)
tEnv.registerTable("MyTable", ds)
val result = tEnv.sql(sqlQuery)
@@ -133,7 +128,7 @@ class SelectITCase(
"8,4\n" + "9,4\n" + "10,4\n" + "11,5\n" + "12,5\n" + "13,5\n" + "14,5\n" + "15,5\n" +
"16,6\n" + "17,6\n" + "18,6\n" + "19,6\n" + "20,6\n" + "21,6\n"
- val results = result.toDataSet[Row](getConfig).collect()
+ val results = result.toDataSet[Row].collect()
TestBaseUtils.compareResultAsText(results.asJava, expected)
}
@@ -141,12 +136,11 @@ class SelectITCase(
def testInvalidFields(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val tEnv = getScalaTableEnvironment
- TranslationContext.reset()
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
val sqlQuery = "SELECT a, foo FROM MyTable"
- val ds = CollectionDataSets.get3TupleDataSet(env).toTable.as('a, 'b, 'c)
+ val ds = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv).as('a, 'b, 'c)
tEnv.registerTable("MyTable", ds)
tEnv.sql(sqlQuery)
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/sql/test/TableWithSQLITCase.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/sql/test/TableWithSQLITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/sql/test/TableWithSQLITCase.scala
index 153a9d0..71334f4 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/sql/test/TableWithSQLITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/sql/test/TableWithSQLITCase.scala
@@ -21,8 +21,7 @@ package org.apache.flink.api.scala.sql.test
import org.apache.flink.api.scala._
import org.apache.flink.api.scala.table._
import org.apache.flink.api.scala.util.CollectionDataSets
-import org.apache.flink.api.table.Row
-import org.apache.flink.api.table.plan.TranslationContext
+import org.apache.flink.api.table.{TableEnvironment, Row}
import org.apache.flink.api.table.test.utils.TableProgramsTestBase
import org.apache.flink.api.table.test.utils.TableProgramsTestBase.TableConfigMode
import org.apache.flink.test.util.MultipleProgramsTestBase.TestExecutionMode
@@ -43,8 +42,7 @@ class TableWithSQLITCase(
def testSQLTable(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val tEnv = getScalaTableEnvironment
- TranslationContext.reset()
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
val ds = CollectionDataSets.get3TupleDataSet(env)
tEnv.registerDataSet("MyTable", ds, 'a, 'b, 'c)
@@ -54,7 +52,7 @@ class TableWithSQLITCase(
val result = tEnv.sql(sqlQuery).select('a.avg, 'b.sum, 'c.count)
val expected = "15,65,12"
- val results = result.toDataSet[Row](getConfig).collect()
+ val results = result.toDataSet[Row].collect()
TestBaseUtils.compareResultAsText(results.asJava, expected)
}
@@ -62,10 +60,9 @@ class TableWithSQLITCase(
def testTableSQLTable(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val tEnv = getScalaTableEnvironment
- TranslationContext.reset()
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
- val ds = CollectionDataSets.get3TupleDataSet(env).as('a, 'b, 'c)
+ val ds = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv, 'a, 'b, 'c)
val t1 = ds.filter('a > 9)
tEnv.registerTable("MyTable", t1)
@@ -75,7 +72,7 @@ class TableWithSQLITCase(
val result = tEnv.sql(sqlQuery).select('a1 + 1, 'b1 - 5, 'c1)
val expected = "16,60,12"
- val results = result.toDataSet[Row](getConfig).collect()
+ val results = result.toDataSet[Row].collect()
TestBaseUtils.compareResultAsText(results.asJava, expected)
}
@@ -83,10 +80,9 @@ class TableWithSQLITCase(
def testMultipleSQLQueries(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val tEnv = getScalaTableEnvironment
- TranslationContext.reset()
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
- val t = CollectionDataSets.get3TupleDataSet(env).as('a, 'b, 'c)
+ val t = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv, 'a, 'b, 'c)
tEnv.registerTable("MyTable", t)
val sqlQuery = "SELECT a as aa FROM MyTable WHERE b = 6"
@@ -97,7 +93,7 @@ class TableWithSQLITCase(
val result2 = tEnv.sql(sqlQuery2)
val expected = "6"
- val results = result2.toDataSet[Row](getConfig).collect()
+ val results = result2.toDataSet[Row].collect()
TestBaseUtils.compareResultAsText(results.asJava, expected)
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/sql/test/UnionITCase.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/sql/test/UnionITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/sql/test/UnionITCase.scala
index 4a031a3..fadc48f 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/sql/test/UnionITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/sql/test/UnionITCase.scala
@@ -21,8 +21,7 @@ package org.apache.flink.api.scala.sql.test
import org.apache.flink.api.scala._
import org.apache.flink.api.scala.table._
import org.apache.flink.api.scala.util.CollectionDataSets
-import org.apache.flink.api.table.Row
-import org.apache.flink.api.table.plan.TranslationContext
+import org.apache.flink.api.table.{TableException, TableEnvironment, Row}
import org.apache.flink.api.table.test.utils.TableProgramsTestBase
import org.apache.flink.api.table.test.utils.TableProgramsTestBase.TableConfigMode
import org.apache.flink.test.util.MultipleProgramsTestBase.TestExecutionMode
@@ -43,8 +42,7 @@ class UnionITCase(
def testUnion(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val tEnv = getScalaTableEnvironment
- TranslationContext.reset()
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
val sqlQuery = "SELECT c FROM t1 UNION ALL (SELECT c FROM t2)"
@@ -56,7 +54,7 @@ class UnionITCase(
val result = tEnv.sql(sqlQuery)
val expected = "Hi\n" + "Hello\n" + "Hello world\n" + "Hi\n" + "Hello\n" + "Hello world\n"
- val results = result.toDataSet[Row](getConfig).collect()
+ val results = result.toDataSet[Row].collect()
TestBaseUtils.compareResultAsText(results.asJava, expected)
}
@@ -65,8 +63,11 @@ class UnionITCase(
def testUnionWithFilter(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val tEnv = getScalaTableEnvironment
- TranslationContext.reset()
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
+
+ if (tEnv.getConfig.getEfficientTypeUsage) {
+ return
+ }
val sqlQuery = "SELECT c FROM (" +
"SELECT * FROM t1 UNION ALL (SELECT a, b, c FROM t2))" +
@@ -89,8 +90,11 @@ class UnionITCase(
def testUnionWithAggregation(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val tEnv = getScalaTableEnvironment
- TranslationContext.reset()
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
+
+ if (tEnv.getConfig.getEfficientTypeUsage) {
+ return
+ }
val sqlQuery = "SELECT count(c) FROM (" +
"SELECT * FROM t1 UNION ALL (SELECT a, b, c FROM t2))"
@@ -106,4 +110,5 @@ class UnionITCase(
val results = result.toDataSet[Row].collect()
TestBaseUtils.compareResultAsText(results.asJava, expected)
}
+
}
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/streaming/test/FilterITCase.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/streaming/test/FilterITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/streaming/test/FilterITCase.scala
index fbfb39e..4760d37 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/streaming/test/FilterITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/streaming/test/FilterITCase.scala
@@ -20,7 +20,7 @@ package org.apache.flink.api.scala.table.streaming.test
import org.apache.flink.api.scala._
import org.apache.flink.api.scala.table._
-import org.apache.flink.api.table.Row
+import org.apache.flink.api.table.{TableEnvironment, Row}
import org.apache.flink.api.table.expressions.Literal
import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase
import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
@@ -40,8 +40,10 @@ class FilterITCase extends StreamingMultipleProgramsTestBase {
* Test simple filter
*/
val env = StreamExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
StreamITCase.testResults = mutable.MutableList()
- val ds = StreamTestData.getSmall3TupleDataStream(env).as('a, 'b, 'c)
+ val ds = StreamTestData.getSmall3TupleDataStream(env).toTable(tEnv, 'a, 'b, 'c)
val filterDs = ds.filter('a === 3)
val results = filterDs.toDataStream[Row]
@@ -58,8 +60,10 @@ class FilterITCase extends StreamingMultipleProgramsTestBase {
* Test all-rejecting filter
*/
val env = StreamExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
StreamITCase.testResults = mutable.MutableList()
- val ds = StreamTestData.getSmall3TupleDataStream(env).as('a, 'b, 'c)
+ val ds = StreamTestData.getSmall3TupleDataStream(env).toTable(tEnv, 'a, 'b, 'c)
val filterDs = ds.filter( Literal(false) )
val results = filterDs.toDataStream[Row]
@@ -75,8 +79,10 @@ class FilterITCase extends StreamingMultipleProgramsTestBase {
* Test all-passing filter
*/
val env = StreamExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
StreamITCase.testResults = mutable.MutableList()
- val ds = StreamTestData.getSmall3TupleDataStream(env).as('a, 'b, 'c)
+ val ds = StreamTestData.getSmall3TupleDataStream(env).toTable(tEnv, 'a, 'b, 'c)
val filterDs = ds.filter( Literal(true) )
val results = filterDs.toDataStream[Row]
@@ -96,8 +102,10 @@ class FilterITCase extends StreamingMultipleProgramsTestBase {
* Test filter on Integer tuple field.
*/
val env = StreamExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
StreamITCase.testResults = mutable.MutableList()
- val ds = StreamTestData.get3TupleDataStream(env).as('a, 'b, 'c)
+ val ds = StreamTestData.get3TupleDataStream(env).toTable(tEnv, 'a, 'b, 'c)
val filterDs = ds.filter( 'a % 2 === 0 )
val results = filterDs.toDataStream[Row]
@@ -118,8 +126,10 @@ class FilterITCase extends StreamingMultipleProgramsTestBase {
* Test filter on Integer tuple field.
*/
val env = StreamExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
StreamITCase.testResults = mutable.MutableList()
- val ds = StreamTestData.get3TupleDataStream(env).as('a, 'b, 'c)
+ val ds = StreamTestData.get3TupleDataStream(env).toTable(tEnv, 'a, 'b, 'c)
val filterDs = ds.filter( 'a % 2 !== 0)
val results = filterDs.toDataStream[Row]
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/streaming/test/SelectITCase.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/streaming/test/SelectITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/streaming/test/SelectITCase.scala
index 83cb7fd..75a9c97 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/streaming/test/SelectITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/streaming/test/SelectITCase.scala
@@ -20,12 +20,10 @@ package org.apache.flink.api.scala.table.streaming.test
import org.apache.flink.api.scala._
import org.apache.flink.api.scala.table._
-import org.apache.flink.api.table.Row
+import org.apache.flink.api.table.{TableEnvironment, Row}
import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase
import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
-import scala.collection.JavaConverters._
import scala.collection.mutable
-import scala.collection.JavaConversions._
import org.junit.Test
import org.junit.Assert._
import org.apache.flink.api.scala.table.streaming.test.utils.StreamITCase
@@ -37,8 +35,9 @@ class StreamSelectITCase extends StreamingMultipleProgramsTestBase {
def testSimpleSelectAll(): Unit = {
val env = StreamExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env)
StreamITCase.testResults = mutable.MutableList()
- val ds = StreamTestData.getSmall3TupleDataStream(env).toStreamTable.select('_1, '_2, '_3)
+ val ds = StreamTestData.getSmall3TupleDataStream(env).toTable(tEnv).select('_1, '_2, '_3)
val results = ds.toDataStream[Row]
results.addSink(new StreamITCase.StringSink)
@@ -55,8 +54,9 @@ class StreamSelectITCase extends StreamingMultipleProgramsTestBase {
def testSelectFirst(): Unit = {
val env = StreamExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env)
StreamITCase.testResults = mutable.MutableList()
- val ds = StreamTestData.getSmall3TupleDataStream(env).toStreamTable.select('_1)
+ val ds = StreamTestData.getSmall3TupleDataStream(env).toTable(tEnv).select('_1)
val results = ds.toDataStream[Row]
results.addSink(new StreamITCase.StringSink)
@@ -71,8 +71,9 @@ class StreamSelectITCase extends StreamingMultipleProgramsTestBase {
// verify ProjectMergeRule.
val env = StreamExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env)
StreamITCase.testResults = mutable.MutableList()
- val ds = StreamTestData.get3TupleDataStream(env).toStreamTable
+ val ds = StreamTestData.get3TupleDataStream(env).toTable(tEnv)
.select('_1 as 'a, '_2 as 'b, '_1 as 'c)
.select('a, 'b)
@@ -91,8 +92,10 @@ class StreamSelectITCase extends StreamingMultipleProgramsTestBase {
def testSimpleSelectAllWithAs(): Unit = {
val env = StreamExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env)
StreamITCase.testResults = mutable.MutableList()
- val ds = StreamTestData.getSmall3TupleDataStream(env).as('a, 'b, 'c).select('a, 'b, 'c)
+ val ds = StreamTestData.getSmall3TupleDataStream(env).toTable(tEnv, 'a, 'b, 'c)
+ .select('a, 'b, 'c)
val results = ds.toDataStream[Row]
results.addSink(new StreamITCase.StringSink)
@@ -109,8 +112,9 @@ class StreamSelectITCase extends StreamingMultipleProgramsTestBase {
def testAsWithToFewFields(): Unit = {
val env = StreamExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env)
StreamITCase.testResults = mutable.MutableList()
- val ds = StreamTestData.get3TupleDataStream(env).as('a, 'b)
+ val ds = StreamTestData.get3TupleDataStream(env).toTable(tEnv, 'a, 'b)
val results = ds.toDataStream[Row]
results.addSink(new StreamITCase.StringSink)
@@ -124,8 +128,9 @@ class StreamSelectITCase extends StreamingMultipleProgramsTestBase {
def testAsWithToManyFields(): Unit = {
val env = StreamExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env)
StreamITCase.testResults = mutable.MutableList()
- val ds = StreamTestData.get3TupleDataStream(env).as('a, 'b, 'c, 'd)
+ val ds = StreamTestData.get3TupleDataStream(env).toTable(tEnv, 'a, 'b, 'c, 'd)
val results = ds.toDataStream[Row]
results.addSink(new StreamITCase.StringSink)
@@ -139,8 +144,9 @@ class StreamSelectITCase extends StreamingMultipleProgramsTestBase {
def testAsWithAmbiguousFields(): Unit = {
val env = StreamExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env)
StreamITCase.testResults = mutable.MutableList()
- val ds = StreamTestData.get3TupleDataStream(env).as('a, 'b, 'b)
+ val ds = StreamTestData.get3TupleDataStream(env).toTable(tEnv, 'a, 'b, 'b)
val results = ds.toDataStream[Row]
results.addSink(new StreamITCase.StringSink)
@@ -155,8 +161,9 @@ class StreamSelectITCase extends StreamingMultipleProgramsTestBase {
def testOnlyFieldRefInAs(): Unit = {
val env = StreamExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env)
StreamITCase.testResults = mutable.MutableList()
- val ds = StreamTestData.get3TupleDataStream(env).as('a, 'b as 'c, 'd)
+ val ds = StreamTestData.get3TupleDataStream(env).toTable(tEnv, 'a, 'b as 'c, 'd)
val results = ds.toDataStream[Row]
results.addSink(new StreamITCase.StringSink)
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/streaming/test/UnionITCase.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/streaming/test/UnionITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/streaming/test/UnionITCase.scala
index ed5e9b5..ae81f3b 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/streaming/test/UnionITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/streaming/test/UnionITCase.scala
@@ -20,7 +20,7 @@ package org.apache.flink.api.scala.table.streaming.test
import org.apache.flink.api.scala._
import org.apache.flink.api.scala.table._
-import org.apache.flink.api.table.Row
+import org.apache.flink.api.table.{TableException, TableEnvironment, Row}
import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase
import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
import scala.collection.JavaConverters._
@@ -36,9 +36,11 @@ class UnionITCase extends StreamingMultipleProgramsTestBase {
@Test
def testUnion(): Unit = {
val env = StreamExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
StreamITCase.testResults = mutable.MutableList()
- val ds1 = StreamTestData.getSmall3TupleDataStream(env).as('a, 'b, 'c)
- val ds2 = StreamTestData.getSmall3TupleDataStream(env).as('a, 'b, 'c)
+ val ds1 = StreamTestData.getSmall3TupleDataStream(env).toTable(tEnv, 'a, 'b, 'c)
+ val ds2 = StreamTestData.getSmall3TupleDataStream(env).toTable(tEnv, 'a, 'b, 'c)
val unionDs = ds1.unionAll(ds2).select('c)
@@ -54,9 +56,11 @@ class UnionITCase extends StreamingMultipleProgramsTestBase {
@Test
def testUnionWithFilter(): Unit = {
val env = StreamExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
StreamITCase.testResults = mutable.MutableList()
- val ds1 = StreamTestData.getSmall3TupleDataStream(env).as('a, 'b, 'c)
- val ds2 = StreamTestData.get5TupleDataStream(env).as('a, 'b, 'd, 'c, 'e)
+ val ds1 = StreamTestData.getSmall3TupleDataStream(env).toTable(tEnv, 'a, 'b, 'c)
+ val ds2 = StreamTestData.get5TupleDataStream(env).toTable(tEnv, 'a, 'b, 'd, 'c, 'e)
val unionDs = ds1.unionAll(ds2.select('a, 'b, 'c)).filter('b < 2).select('c)
@@ -71,9 +75,11 @@ class UnionITCase extends StreamingMultipleProgramsTestBase {
@Test(expected = classOf[IllegalArgumentException])
def testUnionFieldsNameNotOverlap1(): Unit = {
val env = StreamExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
StreamITCase.testResults = mutable.MutableList()
- val ds1 = StreamTestData.getSmall3TupleDataStream(env).as('a, 'b, 'c)
- val ds2 = StreamTestData.get5TupleDataStream(env).as('a, 'b, 'd, 'c, 'e)
+ val ds1 = StreamTestData.getSmall3TupleDataStream(env).toTable(tEnv, 'a, 'b, 'c)
+ val ds2 = StreamTestData.get5TupleDataStream(env).toTable(tEnv, 'a, 'b, 'd, 'c, 'e)
val unionDs = ds1.unionAll(ds2)
@@ -87,9 +93,12 @@ class UnionITCase extends StreamingMultipleProgramsTestBase {
@Test(expected = classOf[IllegalArgumentException])
def testUnionFieldsNameNotOverlap2(): Unit = {
val env = StreamExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
StreamITCase.testResults = mutable.MutableList()
- val ds1 = StreamTestData.getSmall3TupleDataStream(env).as('a, 'b, 'c)
- val ds2 = StreamTestData.get5TupleDataStream(env).as('a, 'b, 'c, 'd, 'e).select('a, 'b, 'c)
+ val ds1 = StreamTestData.getSmall3TupleDataStream(env).toTable(tEnv, 'a, 'b, 'c)
+ val ds2 = StreamTestData.get5TupleDataStream(env).toTable(tEnv, 'a, 'b, 'c, 'd, 'e)
+ .select('a, 'b, 'c)
val unionDs = ds1.unionAll(ds2)
@@ -101,4 +110,17 @@ class UnionITCase extends StreamingMultipleProgramsTestBase {
assertEquals(true, StreamITCase.testResults.isEmpty)
}
+ @Test(expected = classOf[TableException])
+ def testUnionTablesFromDifferentEnvs(): Unit = {
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+ val tEnv1 = TableEnvironment.getTableEnvironment(env)
+ val tEnv2 = TableEnvironment.getTableEnvironment(env)
+
+ val ds1 = StreamTestData.getSmall3TupleDataStream(env).toTable(tEnv1, 'a, 'b, 'c)
+ val ds2 = StreamTestData.getSmall3TupleDataStream(env).toTable(tEnv2, 'a, 'b, 'c)
+
+ // Must fail. Tables are bound to different TableEnvironments.
+ ds1.unionAll(ds2)
+ }
+
}
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/AggregationsITCase.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/AggregationsITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/AggregationsITCase.scala
index abf2735..26cdc76 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/AggregationsITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/AggregationsITCase.scala
@@ -19,7 +19,7 @@
package org.apache.flink.api.scala.table.test
import org.apache.flink.api.table.plan.PlanGenException
-import org.apache.flink.api.table.Row
+import org.apache.flink.api.table.{TableEnvironment, Row}
import org.apache.flink.api.scala._
import org.apache.flink.api.scala.table._
import org.apache.flink.api.scala.util.CollectionDataSets
@@ -38,7 +38,9 @@ class AggregationsITCase(mode: TestExecutionMode) extends MultipleProgramsTestBa
def testAggregationTypes(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val t = CollectionDataSets.get3TupleDataSet(env).toTable
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
+ val t = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv)
.select('_1.sum, '_1.min, '_1.max, '_1.count, '_1.avg)
val results = t.toDataSet[Row].collect()
@@ -50,7 +52,9 @@ class AggregationsITCase(mode: TestExecutionMode) extends MultipleProgramsTestBa
def testAggregationOnNonExistingField(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val t = CollectionDataSets.get3TupleDataSet(env).toTable
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
+ val t = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv)
// Must fail. Field 'foo does not exist.
.select('foo.avg)
}
@@ -59,9 +63,11 @@ class AggregationsITCase(mode: TestExecutionMode) extends MultipleProgramsTestBa
def testWorkingAggregationDataTypes(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
val t = env.fromElements(
(1: Byte, 1: Short, 1, 1L, 1.0f, 1.0d, "Hello"),
- (2: Byte, 2: Short, 2, 2L, 2.0f, 2.0d, "Ciao")).toTable
+ (2: Byte, 2: Short, 2, 2L, 2.0f, 2.0d, "Ciao")).toTable(tEnv)
.select('_1.avg, '_2.avg, '_3.avg, '_4.avg, '_5.avg, '_6.avg, '_7.count)
val expected = "1,1,1,1,1.5,1.5,2"
@@ -73,9 +79,11 @@ class AggregationsITCase(mode: TestExecutionMode) extends MultipleProgramsTestBa
def testProjection(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
val t = env.fromElements(
(1: Byte, 1: Short),
- (2: Byte, 2: Short)).toTable
+ (2: Byte, 2: Short)).toTable(tEnv)
.select('_1.avg, '_1.sum, '_1.count, '_2.avg, '_2.sum)
val expected = "1,3,2,1,3"
@@ -87,7 +95,9 @@ class AggregationsITCase(mode: TestExecutionMode) extends MultipleProgramsTestBa
def testAggregationWithArithmetic(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val t = env.fromElements((1f, "Hello"), (2f, "Ciao")).toTable
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
+ val t = env.fromElements((1f, "Hello"), (2f, "Ciao")).toTable(tEnv)
.select(('_1 + 2).avg + 2, '_2.count + 5)
val expected = "5.5,7"
@@ -99,7 +109,9 @@ class AggregationsITCase(mode: TestExecutionMode) extends MultipleProgramsTestBa
def testAggregationWithTwoCount(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val t = env.fromElements((1f, "Hello"), (2f, "Ciao")).toTable
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
+ val t = env.fromElements((1f, "Hello"), (2f, "Ciao")).toTable(tEnv)
.select('_1.count, '_2.count)
val expected = "2,2"
@@ -111,9 +123,11 @@ class AggregationsITCase(mode: TestExecutionMode) extends MultipleProgramsTestBa
def testAggregationAfterProjection(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
val t = env.fromElements(
(1: Byte, 1: Short, 1, 1L, 1.0f, 1.0d, "Hello"),
- (2: Byte, 2: Short, 2, 2L, 2.0f, 2.0d, "Ciao")).toTable
+ (2: Byte, 2: Short, 2, 2L, 2.0f, 2.0d, "Ciao")).toTable(tEnv)
.select('_1, '_2, '_3)
.select('_1.avg, '_2.sum, '_3.count)
@@ -126,7 +140,9 @@ class AggregationsITCase(mode: TestExecutionMode) extends MultipleProgramsTestBa
def testNonWorkingAggregationDataTypes(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val t = env.fromElements(("Hello", 1)).toTable
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
+ val t = env.fromElements(("Hello", 1)).toTable(tEnv)
// Must fail. Field '_1 is not a numeric type.
.select('_1.sum)
@@ -137,7 +153,9 @@ class AggregationsITCase(mode: TestExecutionMode) extends MultipleProgramsTestBa
def testNoNestedAggregations(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val t = env.fromElements(("Hello", 1)).toTable
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
+ val t = env.fromElements(("Hello", 1)).toTable(tEnv)
// Must fail. Sum aggregation can not be chained.
.select('_2.sum.sum)
}
@@ -146,7 +164,9 @@ class AggregationsITCase(mode: TestExecutionMode) extends MultipleProgramsTestBa
def testSQLStyleAggregations(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val t = CollectionDataSets.get3TupleDataSet(env).as('a, 'b, 'c)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
+ val t = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv, 'a, 'b, 'c)
.select(
"""Sum( a) as a1, a.sum as a2,
|Min (a) as b1, a.min as b2,
@@ -164,13 +184,15 @@ class AggregationsITCase(mode: TestExecutionMode) extends MultipleProgramsTestBa
def testPojoAggregation(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
val input = env.fromElements(
MyWC("hello", 1),
MyWC("hello", 1),
MyWC("ciao", 1),
MyWC("hola", 1),
MyWC("hola", 1))
- val expr = input.toTable
+ val expr = input.toTable(tEnv)
val result = expr
.groupBy('word)
.select('word, 'count.sum as 'count)
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/AsITCase.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/AsITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/AsITCase.scala
deleted file mode 100644
index c2e4e96..0000000
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/AsITCase.scala
+++ /dev/null
@@ -1,151 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.api.scala.table.test
-
-import org.apache.flink.api.scala._
-import org.apache.flink.api.scala.table._
-import org.apache.flink.api.scala.util.CollectionDataSets
-import org.apache.flink.api.table.Row
-import org.apache.flink.api.table.test.utils.TableProgramsTestBase
-import TableProgramsTestBase.TableConfigMode
-import org.apache.flink.api.table.test.utils.TableProgramsTestBase
-import org.apache.flink.test.util.MultipleProgramsTestBase.TestExecutionMode
-import org.apache.flink.test.util.TestBaseUtils
-import org.junit._
-import org.junit.runner.RunWith
-import org.junit.runners.Parameterized
-
-import scala.collection.JavaConverters._
-
-@RunWith(classOf[Parameterized])
-class AsITCase(
- mode: TestExecutionMode,
- configMode: TableConfigMode)
- extends TableProgramsTestBase(mode, configMode) {
-
- @Test
- def testAs(): Unit = {
-
- val env = ExecutionEnvironment.getExecutionEnvironment
- val t = CollectionDataSets.get3TupleDataSet(env)
- .as('a, 'b, 'c)
- .select('a, 'b, 'c)
-
- val expected = "1,1,Hi\n" + "2,2,Hello\n" + "3,2,Hello world\n" +
- "4,3,Hello world, how are you?\n" + "5,3,I am fine.\n" + "6,3,Luke Skywalker\n" +
- "7,4,Comment#1\n" + "8,4,Comment#2\n" + "9,4,Comment#3\n" + "10,4,Comment#4\n" +
- "11,5,Comment#5\n" + "12,5,Comment#6\n" + "13,5,Comment#7\n" + "14,5,Comment#8\n" +
- "15,5,Comment#9\n" + "16,6,Comment#10\n" + "17,6,Comment#11\n" + "18,6,Comment#12\n" +
- "19,6,Comment#13\n" + "20,6,Comment#14\n" + "21,6,Comment#15\n"
- val results = t.toDataSet[Row](getConfig).collect()
- TestBaseUtils.compareResultAsText(results.asJava, expected)
- }
-
- @Test
- def testAsFromCaseClass(): Unit = {
-
- val env = ExecutionEnvironment.getExecutionEnvironment
- val data = List(
- SomeCaseClass("Peter", 28, 4000.00, "Sales"),
- SomeCaseClass("Anna", 56, 10000.00, "Engineering"),
- SomeCaseClass("Lucy", 42, 6000.00, "HR"))
-
- val t = env.fromCollection(data)
- .as('a, 'b, 'c, 'd)
- .select('a, 'b, 'c, 'd)
-
- val expected: String =
- "Peter,28,4000.0,Sales\n" +
- "Anna,56,10000.0,Engineering\n" +
- "Lucy,42,6000.0,HR\n"
- val results = t.toDataSet[Row](getConfig).collect()
- TestBaseUtils.compareResultAsText(results.asJava, expected)
- }
-
- @Test
- def testAsFromAndToCaseClass(): Unit = {
-
- val env = ExecutionEnvironment.getExecutionEnvironment
- val data = List(
- SomeCaseClass("Peter", 28, 4000.00, "Sales"),
- SomeCaseClass("Anna", 56, 10000.00, "Engineering"),
- SomeCaseClass("Lucy", 42, 6000.00, "HR"))
-
- val t = env.fromCollection(data)
- .as('a, 'b, 'c, 'd)
- .select('a, 'b, 'c, 'd)
-
- val expected: String =
- "SomeCaseClass(Peter,28,4000.0,Sales)\n" +
- "SomeCaseClass(Anna,56,10000.0,Engineering)\n" +
- "SomeCaseClass(Lucy,42,6000.0,HR)\n"
- val results = t.toDataSet[SomeCaseClass](getConfig).collect()
- TestBaseUtils.compareResultAsText(results.asJava, expected)
- }
-
- @Test(expected = classOf[IllegalArgumentException])
- def testAsWithToFewFields(): Unit = {
-
- val env = ExecutionEnvironment.getExecutionEnvironment
- CollectionDataSets.get3TupleDataSet(env)
- // Must fail. Number of fields does not match.
- .as('a, 'b)
- }
-
- @Test(expected = classOf[IllegalArgumentException])
- def testAsWithToManyFields(): Unit = {
-
- val env = ExecutionEnvironment.getExecutionEnvironment
- CollectionDataSets.get3TupleDataSet(env)
- // Must fail. Number of fields does not match.
- .as('a, 'b, 'c, 'd)
- }
-
- @Test(expected = classOf[IllegalArgumentException])
- def testAsWithAmbiguousFields(): Unit = {
-
- val env = ExecutionEnvironment.getExecutionEnvironment
- CollectionDataSets.get3TupleDataSet(env)
- // Must fail. Field names not unique.
- .as('a, 'b, 'b)
- }
-
- @Test(expected = classOf[IllegalArgumentException])
- def testAsWithNonFieldReference1(): Unit = {
-
- val env = ExecutionEnvironment.getExecutionEnvironment
- // Must fail. as() can only have field references
- CollectionDataSets.get3TupleDataSet(env)
- .as('a + 1, 'b, 'c)
- }
-
- @Test(expected = classOf[IllegalArgumentException])
- def testAsWithNonFieldReference2(): Unit = {
-
- val env = ExecutionEnvironment.getExecutionEnvironment
- // Must fail. as() can only have field references
- CollectionDataSets.get3TupleDataSet(env)
- .as('a as 'foo, 'b, 'c)
- }
-
-}
-
-case class SomeCaseClass(name: String, age: Int, salary: Double, department: String) {
- def this() { this("", 0, 0.0, "") }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/CalcITCase.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/CalcITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/CalcITCase.scala
index c5d31da..af5e6d7 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/CalcITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/CalcITCase.scala
@@ -21,7 +21,7 @@ package org.apache.flink.api.scala.table.test
import org.apache.flink.api.scala._
import org.apache.flink.api.scala.table._
import org.apache.flink.api.scala.util.CollectionDataSets
-import org.apache.flink.api.table.Row
+import org.apache.flink.api.table.{TableEnvironment, Row}
import org.apache.flink.api.table.test.utils.TableProgramsTestBase
import TableProgramsTestBase.TableConfigMode
import org.apache.flink.test.util.MultipleProgramsTestBase.TestExecutionMode
@@ -39,24 +39,26 @@ class CalcITCase(
@Test
def testSimpleCalc(): Unit = {
-
val env = ExecutionEnvironment.getExecutionEnvironment
- val t = CollectionDataSets.get3TupleDataSet(env).toTable
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
+
+ val t = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv)
.select('_1, '_2, '_3)
.where('_1 < 7)
.select('_1, '_3)
val expected = "1,Hi\n" + "2,Hello\n" + "3,Hello world\n" +
"4,Hello world, how are you?\n" + "5,I am fine.\n" + "6,Luke Skywalker\n"
- val results = t.toDataSet[Row](getConfig).collect()
+ val results = t.toDataSet[Row].collect()
TestBaseUtils.compareResultAsText(results.asJava, expected)
}
@Test
def testCalcWithTwoFilters(): Unit = {
-
val env = ExecutionEnvironment.getExecutionEnvironment
- val t = CollectionDataSets.get3TupleDataSet(env).toTable
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
+
+ val t = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv)
.select('_1, '_2, '_3)
.where('_1 < 7 && '_2 === 3)
.select('_1, '_3)
@@ -64,15 +66,16 @@ class CalcITCase(
.select('_1)
val expected = "4\n"
- val results = t.toDataSet[Row](getConfig).collect()
+ val results = t.toDataSet[Row].collect()
TestBaseUtils.compareResultAsText(results.asJava, expected)
}
@Test
def testCalcWithAggregation(): Unit = {
-
val env = ExecutionEnvironment.getExecutionEnvironment
- val t = CollectionDataSets.get3TupleDataSet(env).toTable
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
+
+ val t = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv)
.select('_1, '_2, '_3)
.where('_1 < 15)
.groupBy('_2)
@@ -80,21 +83,23 @@ class CalcITCase(
.where('cnt > 3)
val expected = "7,4\n" + "11,4\n"
- val results = t.toDataSet[Row](getConfig).collect()
+ val results = t.toDataSet[Row].collect()
TestBaseUtils.compareResultAsText(results.asJava, expected)
}
@Test
def testCalcJoin(): Unit = {
val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = CollectionDataSets.getSmall3TupleDataSet(env).as('a, 'b, 'c)
- val ds2 = CollectionDataSets.get5TupleDataSet(env).as('d, 'e, 'f, 'g, 'h)
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
+
+ val ds1 = CollectionDataSets.getSmall3TupleDataSet(env).toTable(tEnv, 'a, 'b, 'c)
+ val ds2 = CollectionDataSets.get5TupleDataSet(env).toTable(tEnv, 'd, 'e, 'f, 'g, 'h)
val joinT = ds1.select('a, 'b).join(ds2).where('b === 'e).select('a, 'b, 'd, 'e, 'f)
.where('b > 1).select('a, 'd).where('d === 2)
val expected = "2,2\n" + "3,2\n"
- val results = joinT.toDataSet[Row](getConfig).collect()
+ val results = joinT.toDataSet[Row].collect()
TestBaseUtils.compareResultAsText(results.asJava, expected)
}
}
[09/12] flink git commit: [FLINK-3738] [table] Refactor
TableEnvironments. Remove Translators and TranslationContext.
Posted by fh...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/scala/table/TableConversions.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/scala/table/TableConversions.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/scala/table/TableConversions.scala
index f755b17..1fdcbc5 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/scala/table/TableConversions.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/scala/table/TableConversions.scala
@@ -20,41 +20,43 @@ package org.apache.flink.api.scala.table
import org.apache.flink.api.common.typeinfo.TypeInformation
import org.apache.flink.api.scala._
-import org.apache.flink.api.table._
import org.apache.flink.streaming.api.scala.DataStream
+import org.apache.flink.api.table.{Table, TableException}
+import org.apache.flink.api.scala.table.{BatchTableEnvironment => ScalaBatchTableEnv}
+import org.apache.flink.api.scala.table.{StreamTableEnvironment => ScalaStreamTableEnv}
+
/**
- * Methods for converting a [[Table]] to a [[DataSet]] or a [[DataStream]]. A [[Table]] is
- * wrapped in this by the implicit conversions in [[org.apache.flink.api.scala.table]].
- */
+ * Holds methods to convert a [[Table]] into a [[DataSet]] or a [[DataStream]].
+ *
+ * @param table The table to convert.
+ */
class TableConversions(table: Table) {
- /**
- * Converts the [[Table]] to a [[DataSet]] using the default configuration.
- */
+ /** Converts the [[Table]] to a [[DataSet]] of the specified type. */
def toDataSet[T: TypeInformation]: DataSet[T] = {
- new ScalaBatchTranslator().translate[T](table.relNode)
- }
- /**
- * Converts the [[Table]] to a [[DataSet]] using a custom configuration.
- */
- def toDataSet[T: TypeInformation](config: TableConfig): DataSet[T] = {
- new ScalaBatchTranslator(config).translate[T](table.relNode)
+ table.tableEnv match {
+ case tEnv: ScalaBatchTableEnv =>
+ tEnv.toDataSet(table)
+ case _ =>
+ throw new TableException(
+ "Only tables that orginate from Scala DataSets can be converted to Scala DataSets.")
+ }
}
- /**
- * Converts the [[Table]] to a [[DataStream]] using the default configuration.
- */
+ /** Converts the [[Table]] to a [[DataStream]] of the specified type. */
def toDataStream[T: TypeInformation]: DataStream[T] = {
- new ScalaStreamTranslator().translate[T](table.relNode)
- }
- /**
- * Converts the [[Table]] to a [[DataStream]] using a custom configuration.
- */
- def toDataStream[T: TypeInformation](config: TableConfig): DataStream[T] = {
- new ScalaStreamTranslator(config).translate[T](table.relNode)
+ table.tableEnv match {
+ case tEnv: ScalaStreamTableEnv =>
+ tEnv.toDataStream(table)
+ case _ =>
+ throw new TableException(
+ "Only tables that originate from Scala DataStreams " +
+ "can be converted to Scala DataStreams.")
+ }
}
+
}
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/scala/table/TableEnvironment.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/scala/table/TableEnvironment.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/scala/table/TableEnvironment.scala
deleted file mode 100644
index 46164be..0000000
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/scala/table/TableEnvironment.scala
+++ /dev/null
@@ -1,126 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.api.scala.table
-
-import org.apache.flink.api.common.typeinfo.TypeInformation
-import org.apache.flink.api.scala.DataSet
-import org.apache.flink.api.table.expressions.Expression
-import org.apache.flink.api.table.{AbstractTableEnvironment, Table}
-import org.apache.flink.streaming.api.scala.DataStream
-
-/**
- * Environment for working with the Table API.
- *
- * This can be used to convert a [[DataSet]] to a [[Table]] and back again. You
- * can also use the provided methods to create a [[Table]] directly from a data source.
- */
-class TableEnvironment extends AbstractTableEnvironment {
-
- /**
- * Converts the [[DataSet]] to a [[Table]]. The field names can be specified like this:
- *
- * {{{
- * val in: DataSet[(String, Int)] = ...
- * val table = in.as('a, 'b)
- * }}}
- *
- * This results in a [[Table]] that has field `a` of type `String` and field `b`
- * of type `Int`.
- */
- def fromDataSet[T](set: DataSet[T], fields: Expression*): Table = {
- new ScalaBatchTranslator(config).createTable(set, fields.toArray)
- }
-
- /**
- * Transforms the given DataSet to a [[org.apache.flink.api.table.Table]].
- * The fields of the DataSet type are used to name the
- * [[org.apache.flink.api.table.Table]] fields.
- */
- def fromDataSet[T](set: DataSet[T]): Table = {
- new ScalaBatchTranslator(config).createTable(set)
- }
-
- /**
- * Converts the given [[org.apache.flink.api.table.Table]] to
- * a DataSet. The given type must have exactly the same field types and field order as the
- * [[org.apache.flink.api.table.Table]]. Row and tuple types can be mapped by position.
- * POJO types require name equivalence to be mapped correctly as their fields do not have
- * an order.
- */
- def toDataSet[T: TypeInformation](table: Table): DataSet[T] = {
- new ScalaBatchTranslator(config).translate[T](table.relNode)
- }
-
- /**
- * Registers a DataSet under a unique name, so that it can be used in SQL queries.
- * The fields of the DataSet type are used to name the Table fields.
- * @param name the Table name
- * @param dataset the DataSet to register
- */
- def registerDataSet[T](name: String, dataset: DataSet[T]): Unit = {
- registerDataSetInternal(name, dataset.javaSet)
- }
-
- /**
- * Registers a DataSet under a unique name, so that it can be used in SQL queries.
- * The fields of the DataSet type are renamed to the given set of fields.
- *
- * @param name the Table name
- * @param dataset the DataSet to register
- * @param fields the field names expression
- */
- def registerDataSet[T](name: String, dataset: DataSet[T], fields: Expression*): Unit = {
- registerDataSetInternal(name, dataset.javaSet, fields.toArray)
- }
-
- /**
- * Converts the [[DataStream]] to a [[Table]]. The field names can be specified like this:
- *
- * {{{
- * val in: DataStream[(String, Int)] = ...
- * val table = in.as('a, 'b)
- * }}}
- *
- * This results in a [[Table]] that has field `a` of type `String` and field `b`
- * of type `Int`.
- */
- def fromDataStream[T](set: DataStream[T], fields: Expression*): Table = {
- new ScalaStreamTranslator(config).createTable(set, fields.toArray)
- }
-
- /**
- * Transforms the given DataSet to a [[org.apache.flink.api.table.Table]].
- * The fields of the DataSet type are used to name the
- * [[org.apache.flink.api.table.Table]] fields.
- */
- def fromDataStream[T](set: DataStream[T]): Table = {
- new ScalaStreamTranslator(config).createTable(set)
- }
-
- /**
- * Converts the given [[org.apache.flink.api.table.Table]] to
- * a DataStream. The given type must have exactly the same field types and field order as the
- * [[org.apache.flink.api.table.Table]]. Row and tuple types can be mapped by position.
- * POJO types require name equivalence to be mapped correctly as their fields do not have
- * an order.
- */
- def toDataStream[T: TypeInformation](table: Table): DataStream[T] = {
- new ScalaStreamTranslator(config).translate[T](table.relNode)
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/scala/table/package.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/scala/table/package.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/scala/table/package.scala
index 7812d7f..3bce5cf 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/scala/table/package.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/scala/table/package.scala
@@ -20,51 +20,48 @@ package org.apache.flink.api.scala
import org.apache.flink.api.common.typeutils.CompositeType
import org.apache.flink.api.table.{Row, Table}
import scala.language.implicitConversions
-import org.apache.flink.streaming.api.scala.DataStream
+import org.apache.flink.streaming.api.scala._
+
+import scala.reflect.ClassTag
/**
- * == Table API (Scala) ==
- *
- * Importing this package with:
- *
- * {{{
- * import org.apache.flink.api.scala.table._
- * }}}
- *
- * imports implicit conversions for converting a [[DataSet]] and a [[DataStream]] to a
- * [[Table]]. This can be used to perform SQL-like queries on data. Please have
- * a look at [[Table]] to see which operations are supported and
- * [[org.apache.flink.api.scala.table.ImplicitExpressionOperations]] to see how an
- * expression can be specified.
- *
- * When writing a query you can use Scala Symbols to refer to field names. One would
- * refer to field `a` by writing `'a`. Sometimes it is necessary to manually convert a
- * Scala literal to an Expression Literal, in those cases use `Literal`, as in `Literal(3)`.
- *
- * Example:
- *
- * {{{
- * import org.apache.flink.api.scala._
- * import org.apache.flink.api.scala.table._
- *
- * val env = ExecutionEnvironment.getExecutionEnvironment
- * val input = env.fromElements(("Hello", 2), ("Hello", 5), ("Ciao", 3))
- * val result = input.as('word, 'count).groupBy('word).select('word, 'count.avg)
- * result.print()
- *
- * env.execute()
- * }}}
- *
- * A [[Table]] can be converted back to the underlying API
- * representation using `as`:
- *
- * {{{
- * case class Word(word: String, count: Int)
- *
- * val result = in.select(...).as('word, 'count)
- * val set = result.as[Word]
- * }}}
- */
+ * == Table API (Scala) ==
+ *
+ * Importing this package with:
+ *
+ * {{{
+ * import org.apache.flink.api.scala.table._
+ * }}}
+ *
+ * imports implicit conversions for converting a [[DataSet]] and a [[DataStream]] to a
+ * [[Table]]. This can be used to perform SQL-like queries on data. Please have
+ * a look at [[Table]] to see which operations are supported and
+ * [[org.apache.flink.api.scala.table.ImplicitExpressionOperations]] to see how an
+ * expression can be specified.
+ *
+ * When writing a query you can use Scala Symbols to refer to field names. One would
+ * refer to field `a` by writing `'a`. Sometimes it is necessary to manually convert a
+ * Scala literal to an Expression literal, in those cases use `Literal`, as in `Literal(3)`.
+ *
+ * Example:
+ *
+ * {{{
+ * import org.apache.flink.api.scala._
+ * import org.apache.flink.api.scala.table._
+ *
+ * val env = ExecutionEnvironment.getExecutionEnvironment
+ * val tEnv = TableEnvironment.getTableEnvironment(env)
+ *
+ * val input: DataSet[(String, Int)] = env.fromElements(("Hello", 2), ("Hello", 5), ("Ciao", 3))
+ * val result = input
+ * .toTable(tEnv, 'word, 'count)
+ * .groupBy('word)
+ * .select('word, 'count.avg)
+ *
+ * result.print()
+ * }}}
+ *
+ */
package object table extends ImplicitExpressionConversions {
implicit def table2TableConversions(table: Table): TableConversions = {
@@ -72,30 +69,21 @@ package object table extends ImplicitExpressionConversions {
}
implicit def dataSet2DataSetConversions[T](set: DataSet[T]): DataSetConversions[T] = {
- new DataSetConversions[T](set, set.getType)
- }
-
- implicit def table2RowDataSet(
- table: Table): DataSet[Row] = {
- new ScalaBatchTranslator().translate[Row](table.relNode)
+ new DataSetConversions[T](set, set.getType())
}
- implicit def rowDataSet2Table(
- rowDataSet: DataSet[Row]): Table = {
- rowDataSet.toTable
+ implicit def table2RowDataSet(table: Table): DataSet[Row] = {
+ val tableEnv = table.tableEnv.asInstanceOf[BatchTableEnvironment]
+ tableEnv.toDataSet[Row](table)
}
implicit def dataStream2DataStreamConversions[T](set: DataStream[T]): DataStreamConversions[T] = {
- new DataStreamConversions[T](set, set.getType.asInstanceOf[CompositeType[T]])
+ new DataStreamConversions[T](set, set.dataType.asInstanceOf[CompositeType[T]])
}
- implicit def table2RowDataStream(
- table: Table): DataStream[Row] = {
- new ScalaStreamTranslator().translate[Row](table.relNode)
+ implicit def table2RowDataStream(table: Table): DataStream[Row] = {
+ val tableEnv = table.tableEnv.asInstanceOf[StreamTableEnvironment]
+ tableEnv.toDataStream[Row](table)
}
- implicit def rowDataStream2Table(
- rowDataStream: DataStream[Row]): Table = {
- rowDataStream.toStreamTable
- }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/AbstractTableEnvironment.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/AbstractTableEnvironment.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/AbstractTableEnvironment.scala
deleted file mode 100644
index a0f162e..0000000
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/AbstractTableEnvironment.scala
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.api.table
-
-import org.apache.flink.api.java.DataSet
-import org.apache.flink.api.java.table.JavaBatchTranslator
-import org.apache.flink.api.table.expressions.Expression
-import org.apache.flink.api.table.plan.TranslationContext
-import org.apache.flink.api.table.plan.schema.{DataSetTable, TableTable}
-
-class AbstractTableEnvironment {
-
- private[flink] val config = new TableConfig()
-
- /**
- * Returns the table config to define the runtime behavior of the Table API.
- */
- def getConfig = config
-
- /**
- * Registers a Table under a unique name, so that it can be used in SQL queries.
- * @param name the Table name
- * @param table the Table to register
- */
- def registerTable[T](name: String, table: Table): Unit = {
- val tableTable = new TableTable(table.getRelNode())
- TranslationContext.registerTable(tableTable, name)
- }
-
- /**
- * Retrieve a registered Table.
- * @param tableName the name under which the Table has been registered
- * @return the Table object
- */
- @throws[TableException]
- def scan(tableName: String): Table = {
- if (TranslationContext.isRegistered(tableName)) {
- val relBuilder = TranslationContext.getRelBuilder
- relBuilder.scan(tableName)
- new Table(relBuilder.build(), relBuilder)
- }
- else {
- throw new TableException(s"Table \'$tableName\' was not found in the registry.")
- }
- }
-
- private[flink] def registerDataSetInternal[T](name: String, dataset: DataSet[T]): Unit = {
-
- val (fieldNames, fieldIndexes) = TranslationContext.getFieldInfo[T](dataset.getType)
- val dataSetTable = new DataSetTable[T](
- dataset,
- fieldIndexes,
- fieldNames
- )
- TranslationContext.registerTable(dataSetTable, name)
- }
-
- private[flink] def registerDataSetInternal[T](
- name: String, dataset: DataSet[T], fields: Array[Expression]): Unit = {
-
- val (fieldNames, fieldIndexes) = TranslationContext.getFieldInfo[T](
- dataset.getType, fields.toArray)
-
- val dataSetTable = new DataSetTable[T](
- dataset,
- fieldIndexes.toArray,
- fieldNames.toArray
- )
- TranslationContext.registerTable(dataSetTable, name)
- }
-
- /**
- * Execute a SQL query and retrieve the result as a [[Table]].
- * All input [[Table]]s have to be registered in the
- * [[org.apache.flink.api.java.table.TableEnvironment]] with unique names,
- * using [[registerTable()]] or
- * [[org.apache.flink.api.java.table.TableEnvironment.registerDataSet()]]
- *
- * @param query the SQL query
- * @return the result of the SQL query as a [[Table]]
- */
- def sql(query: String): Table = {
- new JavaBatchTranslator(config).translateSQL(query)
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/BatchTableEnvironment.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/BatchTableEnvironment.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/BatchTableEnvironment.scala
new file mode 100644
index 0000000..b6aa229
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/BatchTableEnvironment.scala
@@ -0,0 +1,236 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.table
+
+import java.util.concurrent.atomic.AtomicInteger
+
+import org.apache.calcite.plan.RelOptPlanner.CannotPlanException
+import org.apache.calcite.plan.RelOptUtil
+import org.apache.calcite.sql2rel.RelDecorrelator
+import org.apache.calcite.tools.Programs
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.api.java.DataSet
+import org.apache.flink.api.java.io.DiscardingOutputFormat
+import org.apache.flink.api.java.typeutils.TypeExtractor
+import org.apache.flink.api.table.explain.PlanJsonParser
+import org.apache.flink.api.table.expressions.Expression
+import org.apache.flink.api.table.plan.PlanGenException
+import org.apache.flink.api.table.plan.nodes.dataset.{DataSetRel, DataSetConvention}
+import org.apache.flink.api.table.plan.rules.FlinkRuleSets
+import org.apache.flink.api.table.plan.schema.DataSetTable
+
+/**
+ * The abstract base class for batch TableEnvironments.
+ *
+ * A TableEnvironment can be used to:
+ * - convert a [[DataSet]] to a [[Table]]
+ * - register a [[DataSet]] in the [[org.apache.flink.api.table.TableEnvironment]]'s catalog
+ * - register a [[Table]] in the [[org.apache.flink.api.table.TableEnvironment]]'s catalog
+ * - scan a registered table to obtain a [[Table]]
+ * - specify a SQL query on registered tables to obtain a [[Table]]
+ * - convert a [[Table]] into a [[DataSet]]
+ * - explain the AST and execution plan of a [[Table]]
+ *
+ * @param config The configuration of the TableEnvironment.
+ */
+abstract class BatchTableEnvironment(config: TableConfig) extends TableEnvironment(config) {
+
+ // a counter for unique table names.
+ private val nameCntr: AtomicInteger = new AtomicInteger(0)
+
+ // the naming pattern for internally registered tables.
+ private val internalNamePattern = "^_DataSetTable_[0-9]+$".r
+
+ /**
+ * Checks if the chosen table name is valid.
+ *
+ * @param name The table name to check.
+ */
+ override protected def checkValidTableName(name: String): Unit = {
+ val m = internalNamePattern.findFirstIn(name)
+ m match {
+ case Some(_) =>
+ throw new TableException(s"Illegal Table name. " +
+ s"Please choose a name that does not contain the pattern $internalNamePattern")
+ case None =>
+ }
+ }
+
+ /** Returns a unique table name according to the internal naming pattern. */
+ protected def createUniqueTableName(): String = "_DataSetTable_" + nameCntr.getAndIncrement()
+
+ /**
+ * Scans a registered table and returns the resulting [[Table]].
+ *
+ * The table to scan must be registered in the [[TableEnvironment]]'s catalog.
+ *
+ * @param tableName The name of the table to scan.
+ * @throws TableException if no table is registered under the given name.
+ * @return The scanned table.
+ */
+ @throws[TableException]
+ def scan(tableName: String): Table = {
+
+ if (isRegistered(tableName)) {
+ relBuilder.scan(tableName)
+ new Table(relBuilder.build(), this)
+ }
+ else {
+ throw new TableException(s"Table \'$tableName\' was not found in the registry.")
+ }
+ }
+
+ /**
+ * Evaluates a SQL query on registered tables and retrieves the result as a [[Table]].
+ *
+ * All tables referenced by the query must be registered in the TableEnvironment.
+ *
+ * @param query The SQL query to evaluate.
+ * @return The result of the query as Table.
+ */
+ override def sql(query: String): Table = {
+
+ val planner = new FlinkPlannerImpl(getFrameworkConfig, getPlanner)
+ // parse the sql query
+ val parsed = planner.parse(query)
+ // validate the sql query
+ val validated = planner.validate(parsed)
+ // transform to a relational tree
+ val relational = planner.rel(validated)
+
+ new Table(relational.rel, this)
+ }
+
+ /**
+ * Returns the AST of the specified Table API and SQL queries and the execution plan to compute
+ * the result of the given [[Table]].
+ *
+ * @param table The table for which the AST and execution plan will be returned.
+ * @param extended Flag to include detailed optimizer estimates.
+ */
+ private[flink] def explain(table: Table, extended: Boolean): String = {
+
+ val ast = RelOptUtil.toString(table.relNode)
+ val dataSet = translate[Row](table)(TypeExtractor.createTypeInfo(classOf[Row]))
+ dataSet.output(new DiscardingOutputFormat[Row])
+ val env = dataSet.getExecutionEnvironment
+ val jasonSqlPlan = env.getExecutionPlan
+ val sqlPlan = PlanJsonParser.getSqlExecutionPlan(jasonSqlPlan, extended)
+
+ s"== Abstract Syntax Tree ==\n" +
+ s"$ast\n" +
+ s"== Physical Execution Plan ==\n" +
+ s"$sqlPlan"
+
+ }
+
+ /**
+ * Returns the AST of the specified Table API and SQL queries and the execution plan to compute
+ * the result of the given [[Table]].
+ *
+ * @param table The table for which the AST and execution plan will be returned.
+ */
+ def explain(table: Table): String = explain(table: Table, false)
+
+ /**
+ * Registers a [[DataSet]] as a table under a given name in the [[TableEnvironment]]'s catalog.
+ *
+ * @param name The name under which the table is registered in the catalog.
+ * @param dataSet The [[DataSet]] to register as table in the catalog.
+ * @tparam T the type of the [[DataSet]].
+ */
+ protected def registerDataSetInternal[T](name: String, dataSet: DataSet[T]): Unit = {
+
+ val (fieldNames, fieldIndexes) = getFieldInfo[T](dataSet.getType)
+ val dataSetTable = new DataSetTable[T](
+ dataSet,
+ fieldIndexes,
+ fieldNames
+ )
+ registerTableInternal(name, dataSetTable)
+ }
+
+ /**
+ * Registers a [[DataSet]] as a table under a given name with field names as specified by
+ * field expressions in the [[TableEnvironment]]'s catalog.
+ *
+ * @param name The name under which the table is registered in the catalog.
+ * @param dataSet The [[DataSet]] to register as table in the catalog.
+ * @param fields The field expressions to define the field names of the table.
+ * @tparam T The type of the [[DataSet]].
+ */
+ protected def registerDataSetInternal[T](
+ name: String, dataSet: DataSet[T],
+ fields: Array[Expression]): Unit = {
+
+ val (fieldNames, fieldIndexes) = getFieldInfo[T](dataSet.getType, fields.toArray)
+ val dataSetTable = new DataSetTable[T](
+ dataSet,
+ fieldIndexes.toArray,
+ fieldNames.toArray
+ )
+ registerTableInternal(name, dataSetTable)
+ }
+
+ /**
+ * Translates a [[Table]] into a [[DataSet]].
+ *
+ * The transformation involves optimizing the relational expression tree as defined by
+ * Table API calls and / or SQL queries and generating corresponding [[DataSet]] operators.
+ *
+ * @param table The root node of the relational expression tree.
+ * @param tpe The [[TypeInformation]] of the resulting [[DataSet]].
+ * @tparam A The type of the resulting [[DataSet]].
+ * @return The [[DataSet]] that corresponds to the translated [[Table]].
+ */
+ protected def translate[A](table: Table)(implicit tpe: TypeInformation[A]): DataSet[A] = {
+
+ val relNode = table.getRelNode
+
+ // decorrelate
+ val decorPlan = RelDecorrelator.decorrelateQuery(relNode)
+
+ // optimize the logical Flink plan
+ val optProgram = Programs.ofRules(FlinkRuleSets.DATASET_OPT_RULES)
+ val flinkOutputProps = relNode.getTraitSet.replace(DataSetConvention.INSTANCE).simplify()
+
+ val dataSetPlan = try {
+ optProgram.run(getPlanner, decorPlan, flinkOutputProps)
+ }
+ catch {
+ case e: CannotPlanException =>
+ throw new PlanGenException(
+ s"Cannot generate a valid execution plan for the given query: \n\n" +
+ s"${RelOptUtil.toString(relNode)}\n" +
+ "Please consider filing a bug report.", e)
+ case a: AssertionError =>
+ throw a.getCause
+ }
+
+ dataSetPlan match {
+ case node: DataSetRel =>
+ node.translateToPlan(
+ config,
+ Some(tpe.asInstanceOf[TypeInformation[Any]])
+ ).asInstanceOf[DataSet[A]]
+ case _ => ???
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/StreamTableEnvironment.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/StreamTableEnvironment.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/StreamTableEnvironment.scala
new file mode 100644
index 0000000..7644e6d
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/StreamTableEnvironment.scala
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.table
+
+import java.util.concurrent.atomic.AtomicInteger
+
+import org.apache.calcite.plan.RelOptPlanner.CannotPlanException
+import org.apache.calcite.plan.RelOptUtil
+import org.apache.calcite.sql2rel.RelDecorrelator
+import org.apache.calcite.tools.Programs
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.api.table.expressions.Expression
+import org.apache.flink.api.table.plan.PlanGenException
+import org.apache.flink.api.table.plan.nodes.datastream.{DataStreamRel, DataStreamConvention}
+import org.apache.flink.api.table.plan.rules.FlinkRuleSets
+import org.apache.flink.api.table.plan.schema.DataStreamTable
+import org.apache.flink.streaming.api.datastream.DataStream
+
+/**
+ * The base class for stream TableEnvironments.
+ *
+ * A TableEnvironment can be used to:
+ * - convert [[DataStream]] to a [[Table]]
+ * - register a [[DataStream]] as a table in the catalog
+ * - register a [[Table]] in the catalog
+ * - scan a registered table to obtain a [[Table]]
+ * - specify a SQL query on registered tables to obtain a [[Table]]
+ * - convert a [[Table]] into a [[DataStream]]
+ *
+ * @param config The configuration of the TableEnvironment.
+ */
+abstract class StreamTableEnvironment(config: TableConfig) extends TableEnvironment(config) {
+
+ // a counter for unique table names
+ private val nameCntr: AtomicInteger = new AtomicInteger(0)
+
+ // the naming pattern for internally registered tables.
+ private val internalNamePattern = "^_DataStreamTable_[0-9]+$".r
+
+ /**
+ * Checks if the chosen table name is valid.
+ *
+ * @param name The table name to check.
+ */
+ override protected def checkValidTableName(name: String): Unit = {
+ val m = internalNamePattern.findFirstIn(name)
+ m match {
+ case Some(_) =>
+ throw new TableException(s"Illegal Table name. " +
+ s"Please choose a name that does not contain the pattern $internalNamePattern")
+ case None =>
+ }
+ }
+
+ /** Returns a unique table name according to the internal naming pattern. */
+ protected def createUniqueTableName(): String = "_DataStreamTable_" + nameCntr.getAndIncrement()
+
+ /**
+ * Ingests a registered table and returns the resulting [[Table]].
+ *
+ * The table to ingest must be registered in the [[TableEnvironment]]'s catalog.
+ *
+ * @param tableName The name of the table to ingest.
+ * @throws TableException if no table is registered under the given name.
+ * @return The ingested table.
+ */
+ @throws[TableException]
+ def ingest(tableName: String): Table = {
+
+ if (isRegistered(tableName)) {
+ relBuilder.scan(tableName)
+ new Table(relBuilder.build(), this)
+ }
+ else {
+ throw new TableException(s"Table \'$tableName\' was not found in the registry.")
+ }
+ }
+
+ /**
+ * Evaluates a SQL query on registered tables and retrieves the result as a [[Table]].
+ *
+ * All tables referenced by the query must be registered in the TableEnvironment.
+ *
+ * @param query The SQL query to evaluate.
+ * @return The result of the query as Table.
+ */
+ override def sql(query: String): Table = {
+
+ ???
+ }
+
+ /**
+ * Registers a [[DataStream]] as a table under a given name in the [[TableEnvironment]]'s
+ * catalog.
+ *
+ * @param name The name under which the table is registered in the catalog.
+ * @param dataStream The [[DataStream]] to register as table in the catalog.
+ * @tparam T the type of the [[DataStream]].
+ */
+ protected def registerDataStreamInternal[T](
+ name: String,
+ dataStream: DataStream[T]): Unit = {
+
+ val (fieldNames, fieldIndexes) = getFieldInfo[T](dataStream.getType)
+ val dataStreamTable = new DataStreamTable[T](
+ dataStream,
+ fieldIndexes,
+ fieldNames
+ )
+ registerTableInternal(name, dataStreamTable)
+ }
+
+ /**
+ * Registers a [[DataStream]] as a table under a given name with field names as specified by
+ * field expressions in the [[TableEnvironment]]'s catalog.
+ *
+ * @param name The name under which the table is registered in the catalog.
+ * @param dataStream The [[DataStream]] to register as table in the catalog.
+ * @param fields The field expressions to define the field names of the table.
+ * @tparam T The type of the [[DataStream]].
+ */
+ protected def registerDataStreamInternal[T](
+ name: String,
+ dataStream: DataStream[T],
+ fields: Array[Expression]): Unit = {
+
+ val (fieldNames, fieldIndexes) = getFieldInfo[T](dataStream.getType, fields.toArray)
+ val dataStreamTable = new DataStreamTable[T](
+ dataStream,
+ fieldIndexes.toArray,
+ fieldNames.toArray
+ )
+ registerTableInternal(name, dataStreamTable)
+ }
+
+ /**
+ * Translates a [[Table]] into a [[DataStream]].
+ *
+ * The transformation involves optimizing the relational expression tree as defined by
+ * Table API calls and / or SQL queries and generating corresponding [[DataStream]] operators.
+ *
+ * @param table The root node of the relational expression tree.
+ * @param tpe The [[TypeInformation]] of the resulting [[DataStream]].
+ * @tparam A The type of the resulting [[DataStream]].
+ * @return The [[DataStream]] that corresponds to the translated [[Table]].
+ */
+ protected def translate[A](table: Table)(implicit tpe: TypeInformation[A]): DataStream[A] = {
+
+ val relNode = table.relNode
+
+ // decorrelate
+ val decorPlan = RelDecorrelator.decorrelateQuery(relNode)
+
+ // optimize the logical Flink plan
+ val optProgram = Programs.ofRules(FlinkRuleSets.DATASTREAM_OPT_RULES)
+ val flinkOutputProps = relNode.getTraitSet.replace(DataStreamConvention.INSTANCE).simplify()
+
+ val dataStreamPlan = try {
+ optProgram.run(getPlanner, decorPlan, flinkOutputProps)
+ }
+ catch {
+ case e: CannotPlanException =>
+ throw new PlanGenException(
+ s"Cannot generate a valid execution plan for the given query: \n\n" +
+ s"${RelOptUtil.toString(relNode)}\n" +
+ "Please consider filing a bug report.", e)
+ }
+
+ dataStreamPlan match {
+ case node: DataStreamRel =>
+ node.translateToPlan(
+ config,
+ Some(tpe.asInstanceOf[TypeInformation[Any]])
+ ).asInstanceOf[DataStream[A]]
+ case _ => ???
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/TableEnvironment.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/TableEnvironment.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/TableEnvironment.scala
new file mode 100644
index 0000000..77830ca
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/TableEnvironment.scala
@@ -0,0 +1,358 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.table
+
+import java.util.concurrent.atomic.AtomicInteger
+
+import org.apache.calcite.config.Lex
+import org.apache.calcite.plan.RelOptPlanner
+import org.apache.calcite.schema.SchemaPlus
+import org.apache.calcite.schema.impl.AbstractTable
+import org.apache.calcite.sql.parser.SqlParser
+import org.apache.calcite.tools.{Frameworks, FrameworkConfig, RelBuilder}
+import org.apache.flink.api.common.typeinfo.{AtomicType, TypeInformation}
+import org.apache.flink.api.java.{ExecutionEnvironment => JavaBatchExecEnv}
+import org.apache.flink.api.java.table.{BatchTableEnvironment => JavaBatchTableEnv}
+import org.apache.flink.api.java.table.{StreamTableEnvironment => JavaStreamTableEnv}
+import org.apache.flink.api.java.typeutils.{PojoTypeInfo, TupleTypeInfo}
+import org.apache.flink.api.scala.{ExecutionEnvironment => ScalaBatchExecEnv}
+import org.apache.flink.api.scala.table.{BatchTableEnvironment => ScalaBatchTableEnv}
+import org.apache.flink.api.scala.table.{StreamTableEnvironment => ScalaStreamTableEnv}
+import org.apache.flink.api.scala.typeutils.CaseClassTypeInfo
+import org.apache.flink.api.table.expressions.{Naming, UnresolvedFieldReference, Expression}
+import org.apache.flink.api.table.plan.cost.DataSetCostFactory
+import org.apache.flink.api.table.plan.schema.TableTable
+import org.apache.flink.streaming.api.environment.{StreamExecutionEnvironment => JavaStreamExecEnv}
+import org.apache.flink.streaming.api.scala.{StreamExecutionEnvironment => ScalaStreamExecEnv}
+
+/**
+ * The abstract base class for batch and stream TableEnvironments.
+ *
+ * @param config The configuration of the TableEnvironment
+ */
+abstract class TableEnvironment(val config: TableConfig) {
+
+ // configure sql parser
+ // we use Java lex because back ticks are easier than double quotes in programming
+ // and cases are preserved
+ private val parserConfig = SqlParser
+ .configBuilder()
+ .setLex(Lex.JAVA)
+ .build()
+
+ // the catalog to hold all registered and translated tables
+ private val tables: SchemaPlus = Frameworks.createRootSchema(true)
+
+ // the configuration to create a Calcite planner
+ private val frameworkConfig: FrameworkConfig = Frameworks
+ .newConfigBuilder
+ .defaultSchema(tables)
+ .parserConfig(parserConfig)
+ .costFactory(new DataSetCostFactory)
+ .build
+
+ // the builder for Calcite RelNodes, Calcite's representation of a relational expression tree.
+ protected val relBuilder: RelBuilder = RelBuilder.create(frameworkConfig)
+
+ // the planner instance used to optimize queries of this TableEnvironment
+ private val planner: RelOptPlanner = relBuilder
+ .values(Array("dummy"), new Integer(1))
+ .build().getCluster.getPlanner
+
+ // a counter for unique attribute names
+ private val attrNameCntr: AtomicInteger = new AtomicInteger(0)
+
+ /** Returns the table config to define the runtime behavior of the Table API. */
+ def getConfig = config
+
+ /**
+ * Registers a [[Table]] under a unique name in the TableEnvironment's catalog.
+ * Registered tables can be referenced in SQL queries.
+ *
+ * @param name The name under which the table is registered.
+ * @param table The table to register.
+ */
+ def registerTable(name: String, table: Table): Unit = {
+
+ // check that table belongs to this table environment
+ if (table.tableEnv != this) {
+ throw new TableException(
+ "Only tables that belong to this TableEnvironment can be registered.")
+ }
+
+ checkValidTableName(name)
+
+ val tableTable = new TableTable(table.getRelNode)
+ registerTableInternal(name, tableTable)
+ }
+
+ /**
+ * Evaluates a SQL query on registered tables and retrieves the result as a [[Table]].
+ *
+ * All tables referenced by the query must be registered in the TableEnvironment.
+ *
+ * @param query The SQL query to evaluate.
+ * @return The result of the query as Table.
+ */
+ def sql(query: String): Table
+
+ /**
+ * Registers a Calcite [[AbstractTable]] in the TableEnvironment's catalog.
+ *
+ * @param name The name under which the table is registered.
+ * @param table The table to register in the catalog
+ * @throws TableException if another table is registered under the provided name.
+ */
+ @throws[TableException]
+ protected def registerTableInternal(name: String, table: AbstractTable): Unit = {
+
+ if (isRegistered(name)) {
+ throw new TableException(s"Table \'$name\' already exists. " +
+ s"Please, choose a different name.")
+ } else {
+ tables.add(name, table)
+ }
+ }
+
+ /**
+ * Checks if the chosen table name is valid.
+ *
+ * @param name The table name to check.
+ */
+ protected def checkValidTableName(name: String): Unit
+
+ /**
+ * Checks if a table is registered under the given name.
+ *
+ * @param name The table name to check.
+ * @return true, if a table is registered under the name, false otherwise.
+ */
+ protected def isRegistered(name: String): Boolean = {
+ tables.getTableNames.contains(name)
+ }
+
+ /** Returns a unique temporary attribute name. */
+ private[flink] def createUniqueAttributeName(): String = {
+ "TMP_" + attrNameCntr.getAndIncrement()
+ }
+
+ /** Returns the [[RelBuilder]] of this TableEnvironment. */
+ private[flink] def getRelBuilder: RelBuilder = {
+ relBuilder
+ }
+
+ /** Returns the Calcite [[org.apache.calcite.plan.RelOptPlanner]] of this TableEnvironment. */
+ protected def getPlanner: RelOptPlanner = {
+ planner
+ }
+
+ /** Returns the Calcite [[FrameworkConfig]] of this TableEnvironment. */
+ private[flink] def getFrameworkConfig: FrameworkConfig = {
+ frameworkConfig
+ }
+
+ /**
+ * Returns field names and field positions for a given [[TypeInformation]].
+ *
+ * Field names are automatically extracted for
+ * [[org.apache.flink.api.common.typeutils.CompositeType]].
+ * The method fails if inputType is not a
+ * [[org.apache.flink.api.common.typeutils.CompositeType]].
+ *
+ * @param inputType The TypeInformation extract the field names and positions from.
+ * @tparam A The type of the TypeInformation.
+ * @return A tuple of two arrays holding the field names and corresponding field positions.
+ */
+ protected def getFieldInfo[A](inputType: TypeInformation[A]): (Array[String], Array[Int]) = {
+ val fieldNames: Array[String] = inputType match {
+ case t: TupleTypeInfo[A] => t.getFieldNames
+ case c: CaseClassTypeInfo[A] => c.getFieldNames
+ case p: PojoTypeInfo[A] => p.getFieldNames
+ case tpe =>
+ throw new IllegalArgumentException(
+ s"Type $tpe requires explicit field naming.")
+ }
+ val fieldIndexes = fieldNames.indices.toArray
+ (fieldNames, fieldIndexes)
+ }
+
+ /**
+ * Returns field names and field positions for a given [[TypeInformation]] and [[Array]] of
+ * [[Expression]].
+ *
+ * @param inputType The [[TypeInformation]] against which the [[Expression]]s are evaluated.
+ * @param exprs The expressions that define the field names.
+ * @tparam A The type of the TypeInformation.
+ * @return A tuple of two arrays holding the field names and corresponding field positions.
+ */
+ protected def getFieldInfo[A](
+ inputType: TypeInformation[A],
+ exprs: Array[Expression]): (Array[String], Array[Int]) = {
+
+ val indexedNames: Array[(Int, String)] = inputType match {
+ case a: AtomicType[A] =>
+ if (exprs.length != 1) {
+ throw new IllegalArgumentException("Atomic type may can only have a single field.")
+ }
+ exprs.map {
+ case UnresolvedFieldReference(name) => (0, name)
+ case _ => throw new IllegalArgumentException(
+ "Field reference expression expected.")
+ }
+ case t: TupleTypeInfo[A] =>
+ exprs.zipWithIndex.map {
+ case (UnresolvedFieldReference(name), idx) => (idx, name)
+ case (Naming(UnresolvedFieldReference(origName), name), _) =>
+ val idx = t.getFieldIndex(origName)
+ if (idx < 0) {
+ throw new IllegalArgumentException(s"$origName is not a field of type $t")
+ }
+ (idx, name)
+ case _ => throw new IllegalArgumentException(
+ "Field reference expression or naming expression expected.")
+ }
+ case c: CaseClassTypeInfo[A] =>
+ exprs.zipWithIndex.map {
+ case (UnresolvedFieldReference(name), idx) => (idx, name)
+ case (Naming(UnresolvedFieldReference(origName), name), _) =>
+ val idx = c.getFieldIndex(origName)
+ if (idx < 0) {
+ throw new IllegalArgumentException(s"$origName is not a field of type $c")
+ }
+ (idx, name)
+ case _ => throw new IllegalArgumentException(
+ "Field reference expression or naming expression expected.")
+ }
+ case p: PojoTypeInfo[A] =>
+ exprs.map {
+ case Naming(UnresolvedFieldReference(origName), name) =>
+ val idx = p.getFieldIndex(origName)
+ if (idx < 0) {
+ throw new IllegalArgumentException(s"$origName is not a field of type $p")
+ }
+ (idx, name)
+ case _ => throw new IllegalArgumentException(
+ "Field naming expression expected.")
+ }
+ case tpe => throw new IllegalArgumentException(
+ s"Type $tpe cannot be converted into Table.")
+ }
+
+ val (fieldIndexes, fieldNames) = indexedNames.unzip
+ (fieldNames.toArray, fieldIndexes.toArray)
+ }
+
+}
+
+/**
+ * Object to instantiate a [[TableEnvironment]] depending on the batch or stream execution
+ * environment.
+ */
+object TableEnvironment {
+
+ /**
+ * Returns a [[JavaBatchTableEnv]] for a Java [[JavaBatchExecEnv]].
+ *
+ * @param executionEnvironment The Java batch ExecutionEnvironment.
+ */
+ def getTableEnvironment(executionEnvironment: JavaBatchExecEnv): JavaBatchTableEnv = {
+ new JavaBatchTableEnv(executionEnvironment, new TableConfig())
+ }
+
+ /**
+ * Returns a [[JavaBatchTableEnv]] for a Java [[JavaBatchExecEnv]] and a given [[TableConfig]].
+ *
+ * @param executionEnvironment The Java batch ExecutionEnvironment.
+ * @param tableConfig The TableConfig for the new TableEnvironment.
+ */
+ def getTableEnvironment(
+ executionEnvironment: JavaBatchExecEnv,
+ tableConfig: TableConfig): JavaBatchTableEnv = {
+
+ new JavaBatchTableEnv(executionEnvironment, tableConfig)
+ }
+
+ /**
+ * Returns a [[ScalaBatchTableEnv]] for a Scala [[ScalaBatchExecEnv]].
+ *
+ * @param executionEnvironment The Scala batch ExecutionEnvironment.
+ */
+ def getTableEnvironment(executionEnvironment: ScalaBatchExecEnv): ScalaBatchTableEnv = {
+ new ScalaBatchTableEnv(executionEnvironment, new TableConfig())
+ }
+
+ /**
+ * Returns a [[ScalaBatchTableEnv]] for a Scala [[ScalaBatchExecEnv]] and a given
+ * [[TableConfig]].
+ *
+ * @param executionEnvironment The Scala batch ExecutionEnvironment.
+ * @param tableConfig The TableConfig for the new TableEnvironment.
+ */
+ def getTableEnvironment(
+ executionEnvironment: ScalaBatchExecEnv,
+ tableConfig: TableConfig): ScalaBatchTableEnv = {
+
+ new ScalaBatchTableEnv(executionEnvironment, tableConfig)
+ }
+
+ /**
+ * Returns a [[JavaStreamTableEnv]] for a Java [[JavaStreamExecEnv]].
+ *
+ * @param executionEnvironment The Java StreamExecutionEnvironment.
+ */
+ def getTableEnvironment(executionEnvironment: JavaStreamExecEnv): JavaStreamTableEnv = {
+ new JavaStreamTableEnv(executionEnvironment, new TableConfig())
+ }
+
+ /**
+ * Returns a [[JavaStreamTableEnv]] for a Java [[JavaStreamExecEnv]] and a given [[TableConfig]].
+ *
+ * @param executionEnvironment The Java StreamExecutionEnvironment.
+ * @param tableConfig The TableConfig for the new TableEnvironment.
+ */
+ def getTableEnvironment(
+ executionEnvironment: JavaStreamExecEnv,
+ tableConfig: TableConfig): JavaStreamTableEnv = {
+
+ new JavaStreamTableEnv(executionEnvironment, tableConfig)
+ }
+
+ /**
+ * Returns a [[ScalaStreamTableEnv]] for a Scala stream [[ScalaStreamExecEnv]].
+ *
+ * @param executionEnvironment The Scala StreamExecutionEnvironment.
+ */
+ def getTableEnvironment(executionEnvironment: ScalaStreamExecEnv): ScalaStreamTableEnv = {
+ new ScalaStreamTableEnv(executionEnvironment, new TableConfig())
+ }
+
+ /**
+ * Returns a [[ScalaStreamTableEnv]] for a Scala stream [[ScalaStreamExecEnv]].
+ *
+ * @param executionEnvironment The Scala StreamExecutionEnvironment.
+ * @param tableConfig The TableConfig for the new TableEnvironment.
+ */
+ def getTableEnvironment(
+ executionEnvironment: ScalaStreamExecEnv,
+ tableConfig: TableConfig): ScalaStreamTableEnv = {
+
+ new ScalaStreamTableEnv(executionEnvironment, tableConfig)
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/PlanTranslator.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/PlanTranslator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/PlanTranslator.scala
deleted file mode 100644
index 2a82dc3..0000000
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/PlanTranslator.scala
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.api.table.plan
-
-import org.apache.calcite.rel.RelNode
-import org.apache.flink.api.common.typeinfo.TypeInformation
-import org.apache.flink.api.table.expressions.{ExpressionParser, Expression}
-import org.apache.flink.api.table.Table
-
-import scala.language.reflectiveCalls
-
-/**
- * Base class for translators that transform the logical plan in a [[Table]] to an executable
- * Flink plan and also for creating a [[Table]] from a DataSet.
- */
-abstract class PlanTranslator {
-
- type Representation[A] <: { def getType(): TypeInformation[A] }
-
- /**
- * Translates the given Table API back to the underlying representation, i.e, a DataSet.
- */
- def translate[A](op: RelNode)(implicit tpe: TypeInformation[A]): Representation[A]
-
- /**
- * Creates a [[Table]] from a DataSet (the underlying representation).
- */
- def createTable[A](
- repr: Representation[A],
- fieldIndexes: Array[Int],
- fieldNames: Array[String]): Table
-
- /**
- * Creates a [[Table]] from the given DataSet.
- */
- def createTable[A](repr: Representation[A]): Table = {
-
- val (fieldNames, fieldIndexes) = TranslationContext.getFieldInfo(repr.getType())
- createTable(repr, fieldIndexes, fieldNames)
- }
-
- /**
- * Creates a [[Table]] from the given DataSet while only taking those
- * fields mentioned in the field expressions.
- */
- def createTable[A](repr: Representation[A], expression: String): Table = {
-
- val exprs = ExpressionParser
- .parseExpressionList(expression)
- .toArray
-
- createTable(repr, exprs)
- }
-
- /**
- * Creates a [[Table]] from the given DataSet while only taking those
- * fields mentioned in the field expressions.
- */
- def createTable[A](repr: Representation[A], exprs: Array[Expression]): Table = {
-
- val (fieldNames, fieldIndexes) = TranslationContext.getFieldInfo(repr.getType(), exprs)
- createTable(repr, fieldIndexes.toArray, fieldNames.toArray)
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/RexNodeTranslator.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/RexNodeTranslator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/RexNodeTranslator.scala
index 926e023..0c1db28 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/RexNodeTranslator.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/RexNodeTranslator.scala
@@ -18,8 +18,8 @@
package org.apache.flink.api.table.plan
-import org.apache.calcite.tools.RelBuilder
import org.apache.calcite.tools.RelBuilder.AggCall
+import org.apache.flink.api.table.TableEnvironment
import org.apache.flink.api.table.expressions._
@@ -30,11 +30,15 @@ object RexNodeTranslator {
* these aggregation expressions into Calcite AggCalls, and replaces the original aggregation
* expressions by field accesses expressions.
*/
- def extractAggCalls(exp: Expression, relBuilder: RelBuilder): Pair[Expression, List[AggCall]] = {
+ def extractAggCalls(
+ exp: Expression,
+ tableEnv: TableEnvironment): Pair[Expression, List[AggCall]] = {
+
+ val relBuilder = tableEnv.getRelBuilder
exp match {
case agg: Aggregation =>
- val name = TranslationContext.getUniqueName
+ val name = tableEnv.createUniqueAttributeName()
val aggCall = agg.toAggCall(name)(relBuilder)
val fieldExp = new UnresolvedFieldReference(name)
(fieldExp, List(aggCall))
@@ -45,16 +49,16 @@ object RexNodeTranslator {
case l: LeafExpression =>
(l, Nil)
case u: UnaryExpression =>
- val c = extractAggCalls(u.child, relBuilder)
+ val c = extractAggCalls(u.child, tableEnv)
(u.makeCopy(List(c._1)), c._2)
case b: BinaryExpression =>
- val l = extractAggCalls(b.left, relBuilder)
- val r = extractAggCalls(b.right, relBuilder)
+ val l = extractAggCalls(b.left, tableEnv)
+ val r = extractAggCalls(b.right, tableEnv)
(b.makeCopy(List(l._1, r._1)), l._2 ::: r._2)
// Scalar functions
case c@Call(name, args@_*) =>
- val newArgs = args.map(extractAggCalls(_, relBuilder)).toList
+ val newArgs = args.map(extractAggCalls(_, tableEnv)).toList
(c.makeCopy(name :: newArgs.map(_._1)), newArgs.flatMap(_._2))
case e@AnyRef =>
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/TranslationContext.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/TranslationContext.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/TranslationContext.scala
deleted file mode 100644
index c7d9f18..0000000
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/TranslationContext.scala
+++ /dev/null
@@ -1,219 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.api.table.plan
-
-import java.util.concurrent.atomic.AtomicInteger
-import org.apache.calcite.config.Lex
-import org.apache.calcite.plan.RelOptPlanner
-import org.apache.calcite.schema.impl.AbstractTable
-import org.apache.calcite.schema.SchemaPlus
-import org.apache.calcite.sql.parser.SqlParser
-import org.apache.calcite.tools.{Programs, FrameworkConfig, Frameworks, RelBuilder}
-import org.apache.flink.api.common.typeinfo.{AtomicType, TypeInformation}
-import org.apache.flink.api.java.typeutils.{PojoTypeInfo, TupleTypeInfo}
-import org.apache.flink.api.scala.typeutils.CaseClassTypeInfo
-import org.apache.flink.api.table.TableException
-import org.apache.flink.api.table.expressions.{Naming, UnresolvedFieldReference, Expression}
-import org.apache.flink.api.table.plan.cost.DataSetCostFactory
-import org.apache.flink.api.table.plan.schema.DataStreamTable
-import org.apache.flink.api.table.plan.rules.FlinkRuleSets
-import org.apache.flink.api.table.plan.schema.DataSetTable
-
-object TranslationContext {
-
- private var frameworkConfig: FrameworkConfig = null
- private var relBuilder: RelBuilder = null
- private var tables: SchemaPlus = null
- private var tablesRegistry: Map[String, AbstractTable] = null
- private val nameCntr: AtomicInteger = new AtomicInteger(0)
- private var relOptPlanner: RelOptPlanner = null
-
- reset()
-
- def reset(): Unit = {
-
- // register table in Cascading schema
- tables = Frameworks.createRootSchema(true)
-
- // TODO move this to TableConfig when we implement full SQL support
- // configure sql parser
- // we use Java lex because back ticks are easier than double quotes in programming
- // and cases are preserved
- val parserConfig = SqlParser
- .configBuilder()
- .setLex(Lex.JAVA)
- .build()
-
- // initialize RelBuilder
- frameworkConfig = Frameworks
- .newConfigBuilder
- .defaultSchema(tables)
- .parserConfig(parserConfig)
- .costFactory(new DataSetCostFactory)
- .build
-
- tablesRegistry = Map[String, AbstractTable]()
- relBuilder = RelBuilder.create(frameworkConfig)
- // create a dummy RelNode, in order to retrieve the planner
- val dummyRelNode = relBuilder.values(Array("dummy"), new Integer(1)).build()
- relOptPlanner = dummyRelNode.getCluster.getPlanner
- nameCntr.set(0)
- }
-
- /**
- * Adds a table to the Calcite schema so it can be used by the Table API
- */
- def registerDataSetTable(newTable: DataSetTable[_]): String = {
- val tabName = "_DataSetTable_" + nameCntr.getAndIncrement()
- tables.add(tabName, newTable)
- tabName
- }
-
- /**
- * Adds a table to the Calcite schema and the tables registry,
- * so it can be used by both Table API and SQL statements.
- */
- @throws[TableException]
- def registerTable(table: AbstractTable, name: String): Unit = {
- val illegalPattern = "^_DataSetTable_[0-9]+$".r
- val m = illegalPattern.findFirstIn(name)
- m match {
- case Some(_) =>
- throw new TableException(s"Illegal Table name. " +
- s"Please choose a name that does not contain the pattern $illegalPattern")
- case None => {
- val existingTable = tablesRegistry.get(name)
- existingTable match {
- case Some(_) =>
- throw new TableException(s"Table \'$name\' already exists. " +
- s"Please, choose a different name.")
- case None =>
- tablesRegistry += (name -> table)
- tables.add(name, table)
- }
- }
- }
- }
-
- def isRegistered(name: String): Boolean = {
- val table = tablesRegistry.get(name)
- table match {
- case Some(_) =>
- true
- case None =>
- false
- }
- }
-
- /**
- * Adds a stream Table to the tables registry so it can be used by
- * the streaming Table API.
- */
- def addDataStream(newTable: DataStreamTable[_]): String = {
- val tabName = "_DataStreamTable_" + nameCntr.getAndIncrement()
- tables.add(tabName, newTable)
- tabName
- }
-
- def getUniqueName: String = {
- "TMP_" + nameCntr.getAndIncrement()
- }
-
- def getRelBuilder: RelBuilder = {
- relBuilder
- }
-
- def getPlanner: RelOptPlanner = {
- relOptPlanner
- }
-
- def getFrameworkConfig: FrameworkConfig = {
- frameworkConfig
- }
-
- def getFieldInfo[A](inputType: TypeInformation[A]): (Array[String], Array[Int]) = {
- val fieldNames: Array[String] = inputType match {
- case t: TupleTypeInfo[A] => t.getFieldNames
- case c: CaseClassTypeInfo[A] => c.getFieldNames
- case p: PojoTypeInfo[A] => p.getFieldNames
- case tpe =>
- throw new IllegalArgumentException(
- s"Type $tpe requires explicit field naming with AS.")
- }
- val fieldIndexes = fieldNames.indices.toArray
- (fieldNames, fieldIndexes)
- }
-
- def getFieldInfo[A](
- inputType: TypeInformation[A],
- exprs: Array[Expression]): (Array[String], Array[Int]) = {
-
- val indexedNames: Array[(Int, String)] = inputType match {
- case a: AtomicType[A] =>
- if (exprs.length != 1) {
- throw new IllegalArgumentException("Atomic type may can only have a single field.")
- }
- exprs.map {
- case UnresolvedFieldReference(name) => (0, name)
- case _ => throw new IllegalArgumentException(
- "Field reference expression expected.")
- }
- case t: TupleTypeInfo[A] =>
- exprs.zipWithIndex.map {
- case (UnresolvedFieldReference(name), idx) => (idx, name)
- case (Naming(UnresolvedFieldReference(origName), name), _) =>
- val idx = t.getFieldIndex(origName)
- if (idx < 0) {
- throw new IllegalArgumentException(s"$origName is not a field of type $t")
- }
- (idx, name)
- case _ => throw new IllegalArgumentException(
- "Field reference expression or naming expression expected.")
- }
- case c: CaseClassTypeInfo[A] =>
- exprs.zipWithIndex.map {
- case (UnresolvedFieldReference(name), idx) => (idx, name)
- case (Naming(UnresolvedFieldReference(origName), name), _) =>
- val idx = c.getFieldIndex(origName)
- if (idx < 0) {
- throw new IllegalArgumentException(s"$origName is not a field of type $c")
- }
- (idx, name)
- case _ => throw new IllegalArgumentException(
- "Field reference expression or naming expression expected.")
- }
- case p: PojoTypeInfo[A] =>
- exprs.map {
- case Naming(UnresolvedFieldReference(origName), name) =>
- val idx = p.getFieldIndex(origName)
- if (idx < 0) {
- throw new IllegalArgumentException(s"$origName is not a field of type $p")
- }
- (idx, name)
- case _ => throw new IllegalArgumentException(
- "Field naming expression expected.")
- }
- case tpe => throw new IllegalArgumentException(
- s"Type $tpe cannot be converted into Table.")
- }
-
- val (fieldIndexes, fieldNames) = indexedNames.unzip
- (fieldNames.toArray, fieldIndexes.toArray)
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/table.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/table.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/table.scala
index 7b40c57..f9536a1 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/table.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/table.scala
@@ -17,63 +17,65 @@
*/
package org.apache.flink.api.table
-import org.apache.calcite.plan.RelOptUtil
import org.apache.calcite.rel.RelNode
import org.apache.calcite.rel.`type`.RelDataTypeField
import org.apache.calcite.rel.core.JoinRelType
import org.apache.calcite.rel.logical.LogicalProject
import org.apache.calcite.rex.{RexInputRef, RexLiteral, RexCall, RexNode}
import org.apache.calcite.sql.SqlKind
-import org.apache.calcite.tools.RelBuilder
import org.apache.calcite.tools.RelBuilder.{AggCall, GroupKey}
import org.apache.calcite.util.NlsString
-import org.apache.flink.api.java.io.DiscardingOutputFormat
-import org.apache.flink.api.table.explain.PlanJsonParser
-import org.apache.flink.api.table.plan.{PlanGenException, RexNodeTranslator}
-import RexNodeTranslator.extractAggCalls
-import org.apache.flink.api.table.expressions.{ExpressionParser, Naming, UnresolvedFieldReference, Expression}
-
-import org.apache.flink.api.scala._
-import org.apache.flink.api.scala.table._
+import org.apache.flink.api.table.plan.PlanGenException
+import org.apache.flink.api.table.plan.RexNodeTranslator.extractAggCalls
+import org.apache.flink.api.table.expressions.{ExpressionParser, Naming,
+ UnresolvedFieldReference, Expression}
import scala.collection.mutable
import scala.collection.JavaConverters._
case class BaseTable(
private[flink] val relNode: RelNode,
- private[flink] val relBuilder: RelBuilder)
+ private[flink] val tableEnv: TableEnvironment)
/**
- * The abstraction for writing Table API programs. Similar to how the batch and streaming APIs
- * have [[org.apache.flink.api.scala.DataSet]] and
- * [[org.apache.flink.streaming.api.scala.DataStream]].
+ * A Table is the core component of the Table API.
+ * Similar to how the batch and streaming APIs have DataSet and DataStream,
+ * the Table API is built around [[Table]].
*
- * Use the methods of [[Table]] to transform data. Use
- * [[org.apache.flink.api.java.table.TableEnvironment]] to convert a [[Table]] back to a DataSet
- * or DataStream.
+ * Use the methods of [[Table]] to transform data. Use [[TableEnvironment]] to convert a [[Table]]
+ * back to a DataSet or DataStream.
*
* When using Scala a [[Table]] can also be converted using implicit conversions.
*
* Example:
*
* {{{
- * val table = set.toTable('a, 'b)
+ * val env = ExecutionEnvironment.getExecutionEnvironment
+ * val tEnv = TableEnvironment.getTableEnvironment(env)
+ *
+ * val set: DataSet[(String, Int)] = ...
+ * val table = set.toTable(tEnv, 'a, 'b)
* ...
* val table2 = ...
- * val set = table2.toDataSet[MyType]
+ * val set2: DataSet[MyType] = table2.toDataSet[MyType]
* }}}
*
* Operations such as [[join]], [[select]], [[where]] and [[groupBy]] either take arguments
* in a Scala DSL or as an expression String. Please refer to the documentation for the expression
* syntax.
+ *
+ * @param relNode The root node of the relational Calcite [[RelNode]] tree.
+ * @param tableEnv The [[TableEnvironment]] to which the table is bound.
*/
class Table(
private[flink] override val relNode: RelNode,
- private[flink] override val relBuilder: RelBuilder)
- extends BaseTable(relNode, relBuilder)
+ private[flink] override val tableEnv: TableEnvironment)
+ extends BaseTable(relNode, tableEnv)
{
- def getRelNode(): RelNode = relNode
+ def relBuilder = tableEnv.getRelBuilder
+
+ def getRelNode: RelNode = relNode
/**
* Performs a selection operation. Similar to an SQL SELECT statement. The field expressions
@@ -82,7 +84,7 @@ class Table(
* Example:
*
* {{{
- * in.select('key, 'value.avg + " The average" as 'average, 'other.substring(0, 10))
+ * tab.select('key, 'value.avg + " The average" as 'average)
* }}}
*/
def select(fields: Expression*): Table = {
@@ -93,7 +95,7 @@ class Table(
// separate aggregations and selection expressions
val extractedAggCalls: List[(Expression, List[AggCall])] = fields
- .map(extractAggCalls(_, relBuilder)).toList
+ .map(extractAggCalls(_, tableEnv)).toList
// get aggregation calls
val aggCalls: List[AggCall] = extractedAggCalls.flatMap(_._2)
@@ -113,9 +115,9 @@ class Table(
if(relNode == projected) {
// Calcite's RelBuilder does not translate identity projects even if they rename fields.
// Add a projection ourselves (will be automatically removed by translation rules).
- new Table(createRenamingProject(exprs), relBuilder)
+ new Table(createRenamingProject(exprs), tableEnv)
} else {
- new Table(projected, relBuilder)
+ new Table(projected, tableEnv)
}
}
@@ -127,7 +129,7 @@ class Table(
* Example:
*
* {{{
- * in.select("key, value.avg + " The average" as average, other.substring(0, 10)")
+ * tab.select("key, value.avg + ' The average' as average")
* }}}
*/
def select(fields: String): Table = {
@@ -142,7 +144,7 @@ class Table(
* Example:
*
* {{{
- * in.as('a, 'b)
+ * tab.as('a, 'b)
* }}}
*/
def as(fields: Expression*): Table = {
@@ -169,7 +171,7 @@ class Table(
val exprs = (renamings ++ remaining).map(_.toRexNode(relBuilder))
- new Table(createRenamingProject(exprs), relBuilder)
+ new Table(createRenamingProject(exprs), tableEnv)
}
/**
@@ -179,7 +181,7 @@ class Table(
* Example:
*
* {{{
- * in.as("a, b")
+ * tab.as("a, b")
* }}}
*/
def as(fields: String): Table = {
@@ -194,14 +196,15 @@ class Table(
* Example:
*
* {{{
- * in.filter('name === "Fred")
+ * tab.filter('name === "Fred")
* }}}
*/
def filter(predicate: Expression): Table = {
relBuilder.push(relNode)
relBuilder.filter(predicate.toRexNode(relBuilder))
- new Table(relBuilder.build(), relBuilder)
+
+ new Table(relBuilder.build(), tableEnv)
}
/**
@@ -211,7 +214,7 @@ class Table(
* Example:
*
* {{{
- * in.filter("name = 'Fred'")
+ * tab.filter("name = 'Fred'")
* }}}
*/
def filter(predicate: String): Table = {
@@ -226,7 +229,7 @@ class Table(
* Example:
*
* {{{
- * in.where('name === "Fred")
+ * tab.where('name === "Fred")
* }}}
*/
def where(predicate: Expression): Table = {
@@ -240,7 +243,7 @@ class Table(
* Example:
*
* {{{
- * in.where("name = 'Fred'")
+ * tab.where("name = 'Fred'")
* }}}
*/
def where(predicate: String): Table = {
@@ -254,7 +257,7 @@ class Table(
* Example:
*
* {{{
- * in.groupBy('key).select('key, 'value.avg)
+ * tab.groupBy('key).select('key, 'value.avg)
* }}}
*/
def groupBy(fields: Expression*): GroupedTable = {
@@ -263,7 +266,7 @@ class Table(
val groupExpr = fields.map(_.toRexNode(relBuilder)).toIterable.asJava
val groupKey = relBuilder.groupKey(groupExpr)
- new GroupedTable(relBuilder.build(), relBuilder, groupKey)
+ new GroupedTable(relBuilder.build(), tableEnv, groupKey)
}
/**
@@ -273,7 +276,7 @@ class Table(
* Example:
*
* {{{
- * in.groupBy("key").select("key, value.avg")
+ * tab.groupBy("key").select("key, value.avg")
* }}}
*/
def groupBy(fields: String): GroupedTable = {
@@ -287,13 +290,13 @@ class Table(
* Example:
*
* {{{
- * in.select("key, value").distinct()
+ * tab.select("key, value").distinct()
* }}}
*/
def distinct(): Table = {
relBuilder.push(relNode)
relBuilder.distinct()
- new Table(relBuilder.build(), relBuilder)
+ new Table(relBuilder.build(), tableEnv)
}
/**
@@ -301,6 +304,8 @@ class Table(
* operations must not overlap, use [[as]] to rename fields if necessary. You can use
* where and select clauses after a join to further specify the behaviour of the join.
*
+ * Note: Both tables must be bound to the same [[TableEnvironment]].
+ *
* Example:
*
* {{{
@@ -309,6 +314,11 @@ class Table(
*/
def join(right: Table): Table = {
+ // check that right table belongs to the same TableEnvironment
+ if (right.tableEnv != this.tableEnv) {
+ throw new TableException("Only tables from the same TableEnvironment can be joined.")
+ }
+
// check that join inputs do not have overlapping field names
val leftFields = relNode.getRowType.getFieldNames.asScala.toSet
val rightFields = right.relNode.getRowType.getFieldNames.asScala.toSet
@@ -321,13 +331,15 @@ class Table(
relBuilder.join(JoinRelType.INNER, relBuilder.literal(true))
val join = relBuilder.build()
- new Table(join, relBuilder)
+ new Table(join, tableEnv)
}
/**
* Union two [[Table]]s. Similar to an SQL UNION ALL. The fields of the two union operations
* must fully overlap.
*
+ * Note: Both tables must be bound to the same [[TableEnvironment]].
+ *
* Example:
*
* {{{
@@ -336,6 +348,11 @@ class Table(
*/
def unionAll(right: Table): Table = {
+ // check that right table belongs to the same TableEnvironment
+ if (right.tableEnv != this.tableEnv) {
+ throw new TableException("Only tables from the same TableEnvironment can be unioned.")
+ }
+
val leftRowType: List[RelDataTypeField] = relNode.getRowType.getFieldList.asScala.toList
val rightRowType: List[RelDataTypeField] = right.relNode.getRowType.getFieldList.asScala.toList
@@ -355,31 +372,9 @@ class Table(
relBuilder.push(right.relNode)
relBuilder.union(true)
- new Table(relBuilder.build(), relBuilder)
+ new Table(relBuilder.build(), tableEnv)
}
- /**
- * Get the process of the sql parsing, print AST and physical execution plan.The AST
- * show the structure of the supplied statement. The execution plan shows how the table
- * referenced by the statement will be scanned.
- */
- private[flink] def explain(extended: Boolean): String = {
-
- val ast = RelOptUtil.toString(relNode)
- val dataSet = this.toDataSet[Row]
- dataSet.output(new DiscardingOutputFormat[Row])
- val env = dataSet.getExecutionEnvironment
- val jasonSqlPlan = env.getExecutionPlan()
- val sqlPlan = PlanJsonParser.getSqlExecutionPlan(jasonSqlPlan, extended)
- val result = "== Abstract Syntax Tree ==\n" + ast + "\n" + "== Physical Execution Plan ==" +
- "\n" + sqlPlan
- return result
-
- ""
- }
-
- def explain(): String = explain(false)
-
private def createRenamingProject(exprs: Seq[RexNode]): LogicalProject = {
val names = exprs.map{ e =>
@@ -422,19 +417,28 @@ class Table(
}
+/**
+ * A table that has been grouped on a set of grouping keys.
+ *
+ * @param relNode The root node of the relational Calcite [[RelNode]] tree.
+ * @param tableEnv The [[TableEnvironment]] to which the table is bound.
+ * @param groupKey The Calcite [[GroupKey]] of this table.
+ */
class GroupedTable(
private[flink] override val relNode: RelNode,
- private[flink] override val relBuilder: RelBuilder,
- private[flink] val groupKey: GroupKey) extends BaseTable(relNode, relBuilder) {
+ private[flink] override val tableEnv: TableEnvironment,
+ private[flink] val groupKey: GroupKey) extends BaseTable(relNode, tableEnv) {
+
+ def relBuilder = tableEnv.getRelBuilder
/**
- * Performs a selection operation. Similar to an SQL SELECT statement. The field expressions
- * can contain complex expressions and aggregations.
+ * Performs a selection operation on a grouped table. Similar to an SQL SELECT statement.
+ * The field expressions can contain complex expressions and aggregations.
*
* Example:
*
* {{{
- * in.select('key, 'value.avg + " The average" as 'average, 'other.substring(1, 10))
+ * tab.groupBy('key).select('key, 'value.avg + " The average" as 'average)
* }}}
*/
def select(fields: Expression*): Table = {
@@ -443,7 +447,7 @@ class GroupedTable(
// separate aggregations and selection expressions
val extractedAggCalls: List[(Expression, List[AggCall])] = fields
- .map(extractAggCalls(_, relBuilder)).toList
+ .map(extractAggCalls(_, tableEnv)).toList
// get aggregation calls
val aggCalls: List[AggCall] = extractedAggCalls.flatMap(_._2)
@@ -463,17 +467,17 @@ class GroupedTable(
relBuilder.project(exprs.toIterable.asJava)
- new Table(relBuilder.build(), relBuilder)
+ new Table(relBuilder.build(), tableEnv)
}
/**
- * Performs a selection operation. Similar to an SQL SELECT statement. The field expressions
- * can contain complex expressions and aggregations.
+ * Performs a selection operation on a grouped table. Similar to an SQL SELECT statement.
+ * The field expressions can contain complex expressions and aggregations.
*
* Example:
*
* {{{
- * in.select("key, value.avg + " The average" as average, other.substring(1, 10)")
+ * tab.groupBy("key").select("key, value.avg + " The average" as average")
* }}}
*/
def select(fields: String): Table = {
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/main/scala/org/apache/flink/examples/scala/TPCHQuery3Table.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/examples/scala/TPCHQuery3Table.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/examples/scala/TPCHQuery3Table.scala
index f527a3c..56f32bd 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/examples/scala/TPCHQuery3Table.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/examples/scala/TPCHQuery3Table.scala
@@ -17,6 +17,7 @@
*/
package org.apache.flink.examples.scala
+import org.apache.flink.api.table.TableEnvironment
import org.apache.flink.api.table.expressions.Literal
import org.apache.flink.api.scala._
import org.apache.flink.api.scala.table._
@@ -80,17 +81,18 @@ object TPCHQuery3Table {
// get execution environment
val env = ExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env)
val lineitems = getLineitemDataSet(env)
- .filter( l => dateFormat.parse(l.shipDate).after(date) )
+ .filter( l => dateFormat.parse(l.shipDate).after(date) ).toTable(tEnv)
.as('id, 'extdPrice, 'discount, 'shipDate)
- val customers = getCustomerDataSet(env)
+ val customers = getCustomerDataSet(env).toTable(tEnv)
.as('id, 'mktSegment)
.filter( 'mktSegment === "AUTOMOBILE" )
val orders = getOrdersDataSet(env)
- .filter( o => dateFormat.parse(o.orderDate).before(date) )
+ .filter( o => dateFormat.parse(o.orderDate).before(date) ).toTable(tEnv)
.as('orderId, 'custId, 'orderDate, 'shipPrio)
val items =
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/main/scala/org/apache/flink/examples/scala/WordCountTable.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/examples/scala/WordCountTable.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/examples/scala/WordCountTable.scala
index cac9590..3c53fc4 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/examples/scala/WordCountTable.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/examples/scala/WordCountTable.scala
@@ -20,6 +20,7 @@ package org.apache.flink.examples.scala
import org.apache.flink.api.scala._
import org.apache.flink.api.scala.table._
+import org.apache.flink.api.table.TableEnvironment
/**
* Simple example for demonstrating the use of the Table API for a Word Count.
@@ -32,9 +33,10 @@ object WordCountTable {
// set up execution environment
val env = ExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env)
val input = env.fromElements(WC("hello", 1), WC("hello", 1), WC("ciao", 1))
- val expr = input.toTable
+ val expr = input.toTable(tEnv)
val result = expr
.groupBy('word)
.select('word, 'count.sum as 'count)
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/AggregationsITCase.java
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/AggregationsITCase.java b/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/AggregationsITCase.java
index 597daea..d3856ec 100644
--- a/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/AggregationsITCase.java
+++ b/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/AggregationsITCase.java
@@ -41,12 +41,12 @@ import org.apache.flink.api.table.ExpressionParserException;
import org.apache.flink.api.table.Row;
import org.apache.flink.api.table.Table;
import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.table.TableEnvironment;
+import org.apache.flink.api.java.table.BatchTableEnvironment;
import org.apache.flink.api.java.operators.DataSource;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.tuple.Tuple7;
+import org.apache.flink.api.table.TableEnvironment;
import org.apache.flink.api.table.plan.PlanGenException;
-import org.apache.flink.examples.java.JavaTableExample;
import org.apache.flink.test.javaApiOperators.util.CollectionDataSets;
import org.apache.flink.test.util.MultipleProgramsTestBase;
import org.junit.Test;
@@ -66,7 +66,7 @@ public class AggregationsITCase extends MultipleProgramsTestBase {
@Test
public void testAggregationTypes() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = new TableEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
Table table = tableEnv.fromDataSet(CollectionDataSets.get3TupleDataSet(env));
@@ -81,7 +81,7 @@ public class AggregationsITCase extends MultipleProgramsTestBase {
@Test(expected = IllegalArgumentException.class)
public void testAggregationOnNonExistingField() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = new TableEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
Table table =
tableEnv.fromDataSet(CollectionDataSets.get3TupleDataSet(env));
@@ -98,7 +98,7 @@ public class AggregationsITCase extends MultipleProgramsTestBase {
@Test
public void testWorkingAggregationDataTypes() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = new TableEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
DataSource<Tuple7<Byte, Short, Integer, Long, Float, Double, String>> input =
env.fromElements(
@@ -119,7 +119,7 @@ public class AggregationsITCase extends MultipleProgramsTestBase {
@Test
public void testAggregationWithArithmetic() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = new TableEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
DataSource<Tuple2<Float, String>> input =
env.fromElements(
@@ -141,7 +141,7 @@ public class AggregationsITCase extends MultipleProgramsTestBase {
@Test
public void testAggregationWithTwoCount() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = new TableEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
DataSource<Tuple2<Float, String>> input =
env.fromElements(
@@ -163,7 +163,7 @@ public class AggregationsITCase extends MultipleProgramsTestBase {
@Test(expected = PlanGenException.class)
public void testNonWorkingDataTypes() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = new TableEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
DataSource<Tuple2<Float, String>> input = env.fromElements(new Tuple2<>(1f, "Hello"));
@@ -183,7 +183,7 @@ public class AggregationsITCase extends MultipleProgramsTestBase {
@Test(expected = ExpressionParserException.class)
public void testNoNestedAggregation() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = new TableEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
DataSource<Tuple2<Float, String>> input = env.fromElements(new Tuple2<>(1f, "Hello"));
@@ -203,7 +203,7 @@ public class AggregationsITCase extends MultipleProgramsTestBase {
@Test
public void testPojoAggregation() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = new TableEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
DataSet<WC> input = env.fromElements(
new WC("Hello", 1),
new WC("Ciao", 1),
[03/12] flink git commit: [FLINK-3700] [core] Remove Guava dependency
from flink-core
Posted by fh...@apache.org.
[FLINK-3700] [core] Remove Guava dependency from flink-core
Almost all Guava functionality used within flink-core has corresponding
utils in Flink's codebase, or the JDK library.
This replaces the Guava code as follows
- Preconditions calls by Flink's Preconditions class
- Collection utils by simple Java Collection calls
- Iterator's by Flink's Union Iterator
- Files by simple util methods arount java.nio.Files
- InetAddresses IPv6 encoding code has been adapted into Flink's NetUtils (with attribution comments)
Some util classes where moved from flink-runtime to flink-core.
This closes #1854
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/760a0d9e
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/760a0d9e
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/760a0d9e
Branch: refs/heads/master
Commit: 760a0d9e7fd9fa88e9f7408b137d78df384d764f
Parents: 1b93b32
Author: Stephan Ewen <se...@apache.org>
Authored: Tue Apr 5 15:18:32 2016 +0200
Committer: Fabian Hueske <fh...@apache.org>
Committed: Fri Apr 15 19:38:14 2016 +0200
----------------------------------------------------------------------
.../storm/tests/StormFieldsGroupingITCase.java | 2 +-
flink-core/pom.xml | 6 -
.../java/org/apache/flink/api/common/Plan.java | 5 +-
.../org/apache/flink/api/common/TaskInfo.java | 4 +-
.../util/AbstractRuntimeUDFContext.java | 9 +-
.../api/common/io/DelimitedInputFormat.java | 17 +-
.../flink/api/common/io/FileInputFormat.java | 28 +--
.../api/common/io/GenericCsvInputFormat.java | 43 ++--
.../common/operators/GenericDataSinkBase.java | 21 +-
.../apache/flink/api/common/operators/Keys.java | 19 +-
.../base/GroupCombineOperatorBase.java | 6 +-
.../operators/base/GroupReduceOperatorBase.java | 6 +-
.../api/common/operators/util/FieldList.java | 6 +-
.../operators/util/UserCodeObjectWrapper.java | 7 +-
.../api/common/typeinfo/BasicArrayTypeInfo.java | 10 +-
.../api/common/typeinfo/BasicTypeInfo.java | 8 +-
.../api/common/typeinfo/FractionalTypeInfo.java | 19 +-
.../api/common/typeinfo/IntegerTypeInfo.java | 25 ++-
.../api/common/typeinfo/NumericTypeInfo.java | 29 +--
.../common/typeinfo/PrimitiveArrayTypeInfo.java | 22 +-
.../api/common/typeutils/CompositeType.java | 4 +-
.../common/typeutils/base/EnumSerializer.java | 5 +-
.../typeutils/base/GenericArraySerializer.java | 7 +-
.../flink/api/java/typeutils/EnumTypeInfo.java | 7 +-
.../api/java/typeutils/GenericTypeInfo.java | 4 +-
.../api/java/typeutils/ObjectArrayTypeInfo.java | 15 +-
.../flink/api/java/typeutils/PojoField.java | 7 +-
.../flink/api/java/typeutils/PojoTypeInfo.java | 40 ++--
.../flink/api/java/typeutils/TupleTypeInfo.java | 32 ++-
.../api/java/typeutils/TupleTypeInfoBase.java | 6 +-
.../flink/api/java/typeutils/TypeExtractor.java | 8 +-
.../flink/api/java/typeutils/ValueTypeInfo.java | 12 +-
.../api/java/typeutils/WritableTypeInfo.java | 11 +-
.../java/typeutils/runtime/AvroSerializer.java | 7 +-
.../runtime/CopyableValueSerializer.java | 4 +-
.../java/typeutils/runtime/PojoSerializer.java | 10 +-
.../typeutils/runtime/TupleSerializerBase.java | 6 +-
.../java/typeutils/runtime/ValueSerializer.java | 5 +-
.../typeutils/runtime/kryo/KryoSerializer.java | 8 +-
.../org/apache/flink/types/StringValue.java | 14 +-
.../java/org/apache/flink/util/FileUtils.java | 91 ++++++++
.../java/org/apache/flink/util/IOUtils.java | 223 +++++++++++++++++++
.../java/org/apache/flink/util/MathUtils.java | 175 +++++++++++++++
.../java/org/apache/flink/util/NetUtils.java | 86 ++++++-
.../org/apache/flink/util/Preconditions.java | 57 ++++-
.../org/apache/flink/util/UnionIterator.java | 102 +++++++++
.../org/apache/flink/util/XORShiftRandom.java | 4 +-
.../base/OuterJoinOperatorBaseTest.java | 4 +-
.../flink/api/java/tuple/TupleGenerator.java | 7 +-
.../java/typeutils/PojoTypeExtractionTest.java | 13 +-
.../typeutils/runtime/PojoSerializerTest.java | 8 +-
.../runtime/PojoSubclassSerializerTest.java | 5 +-
.../SubclassFromInterfaceSerializerTest.java | 5 +-
.../apache/flink/testutils/junit/RetryRule.java | 5 +-
.../org/apache/flink/util/MathUtilTest.java | 102 +++++++++
.../org/apache/flink/util/NetUtilsTest.java | 4 -
.../apache/flink/util/UnionIteratorTest.java | 143 ++++++++++++
.../apache/flink/runtime/blob/BlobUtils.java | 2 +-
.../flink/runtime/blob/FileSystemBlobStore.java | 2 +-
.../checkpoint/FileSystemStateStore.java | 2 +-
.../flink/runtime/filecache/FileCache.java | 2 +-
.../runtime/io/disk/FileChannelInputView.java | 2 +-
.../runtime/io/disk/RandomAccessInputView.java | 2 +-
.../runtime/io/disk/RandomAccessOutputView.java | 2 +-
.../io/disk/SeekableFileChannelInputView.java | 2 +-
.../io/disk/SimpleCollectingOutputView.java | 2 +-
.../flink/runtime/memory/MemoryManager.java | 2 +-
.../operators/hash/CompactingHashTable.java | 2 +-
.../runtime/operators/hash/HashPartition.java | 2 +-
.../operators/hash/MutableHashTable.java | 2 +-
.../operators/shipping/OutputEmitter.java | 2 +-
.../apache/flink/runtime/util/FileUtils.java | 61 -----
.../org/apache/flink/runtime/util/IOUtils.java | 213 ------------------
.../apache/flink/runtime/util/MathUtils.java | 176 ---------------
.../flink/runtime/util/UnionIterator.java | 100 ---------
.../FileSystemStateStorageHelper.java | 2 +-
.../flink/runtime/taskmanager/TaskManager.scala | 2 +-
.../runtime/testutils/CommonTestUtils.java | 2 +-
.../apache/flink/runtime/util/MathUtilTest.java | 101 ---------
.../flink/runtime/util/UnionIteratorTest.java | 142 ------------
.../source/SocketTextStreamFunction.java | 2 +-
...ractAlignedProcessingTimeWindowOperator.java | 2 +-
.../windowing/AccumulatingKeyedTimePanes.java | 2 +-
.../runtime/operators/windowing/KeyMap.java | 2 +-
.../runtime/partitioner/HashPartitioner.java | 2 +-
.../apache/flink/streaming/api/IterateTest.java | 2 +-
.../streaming/api/StreamingOperatorsITCase.java | 2 +-
.../api/scala/StreamingOperatorsITCase.scala | 2 +-
88 files changed, 1294 insertions(+), 1087 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/StormFieldsGroupingITCase.java
----------------------------------------------------------------------
diff --git a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/StormFieldsGroupingITCase.java b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/StormFieldsGroupingITCase.java
index 1875ecb..b43b24d 100644
--- a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/StormFieldsGroupingITCase.java
+++ b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/StormFieldsGroupingITCase.java
@@ -21,7 +21,7 @@ import backtype.storm.Config;
import backtype.storm.topology.TopologyBuilder;
import backtype.storm.tuple.Fields;
-import org.apache.flink.runtime.util.MathUtils;
+import org.apache.flink.util.MathUtils;
import org.apache.flink.storm.api.FlinkLocalCluster;
import org.apache.flink.storm.api.FlinkTopology;
import org.apache.flink.storm.tests.operators.FiniteRandomSpout;
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-core/pom.xml
----------------------------------------------------------------------
diff --git a/flink-core/pom.xml b/flink-core/pom.xml
index 59b620b..eb55bdd 100644
--- a/flink-core/pom.xml
+++ b/flink-core/pom.xml
@@ -70,12 +70,6 @@ under the License.
<artifactId>${shading-artifact.name}</artifactId>
<version>${project.version}</version>
</dependency>
-
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <version>${guava.version}</version>
- </dependency>
<!-- test depedencies -->
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-core/src/main/java/org/apache/flink/api/common/Plan.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/Plan.java b/flink-core/src/main/java/org/apache/flink/api/common/Plan.java
index 30f2c2f..0235af0 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/Plan.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/Plan.java
@@ -18,9 +18,6 @@
package org.apache.flink.api.common;
-import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkNotNull;
-
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collection;
@@ -43,6 +40,8 @@ import org.apache.flink.core.fs.Path;
import org.apache.flink.util.Visitable;
import org.apache.flink.util.Visitor;
+import static org.apache.flink.util.Preconditions.checkNotNull;
+import static org.apache.flink.util.Preconditions.checkArgument;
/**
* This class represents Flink programs, in the form of dataflow plans.
*
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-core/src/main/java/org/apache/flink/api/common/TaskInfo.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/TaskInfo.java b/flink-core/src/main/java/org/apache/flink/api/common/TaskInfo.java
index 6482cde..ac87e74 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/TaskInfo.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/TaskInfo.java
@@ -20,8 +20,8 @@ package org.apache.flink.api.common;
import org.apache.flink.annotation.Internal;
-import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkNotNull;
+import static org.apache.flink.util.Preconditions.checkNotNull;
+import static org.apache.flink.util.Preconditions.checkArgument;
/**
* Encapsulates task-specific information: name, index of subtask, parallelism and attempt number.
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-core/src/main/java/org/apache/flink/api/common/functions/util/AbstractRuntimeUDFContext.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/util/AbstractRuntimeUDFContext.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/util/AbstractRuntimeUDFContext.java
index afe9f77..74b78df 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/functions/util/AbstractRuntimeUDFContext.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/functions/util/AbstractRuntimeUDFContext.java
@@ -23,7 +23,6 @@ import java.util.Collections;
import java.util.Map;
import java.util.concurrent.Future;
-import com.google.common.base.Preconditions;
import org.apache.flink.annotation.PublicEvolving;
import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.TaskInfo;
@@ -45,6 +44,8 @@ import org.apache.flink.api.common.state.ValueStateDescriptor;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.core.fs.Path;
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
/**
* A standalone implementation of the {@link RuntimeContext}, created by runtime UDF operators.
*/
@@ -66,11 +67,11 @@ public abstract class AbstractRuntimeUDFContext implements RuntimeContext {
ExecutionConfig executionConfig,
Map<String, Accumulator<?,?>> accumulators,
Map<String, Future<Path>> cpTasks) {
- this.taskInfo = Preconditions.checkNotNull(taskInfo);
+ this.taskInfo = checkNotNull(taskInfo);
this.userCodeClassLoader = userCodeClassLoader;
this.executionConfig = executionConfig;
- this.distributedCache = new DistributedCache(Preconditions.checkNotNull(cpTasks));
- this.accumulators = Preconditions.checkNotNull(accumulators);
+ this.distributedCache = new DistributedCache(checkNotNull(cpTasks));
+ this.accumulators = checkNotNull(accumulators);
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-core/src/main/java/org/apache/flink/api/common/io/DelimitedInputFormat.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/io/DelimitedInputFormat.java b/flink-core/src/main/java/org/apache/flink/api/common/io/DelimitedInputFormat.java
index f6b6d49..243e2a4 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/io/DelimitedInputFormat.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/io/DelimitedInputFormat.java
@@ -18,12 +18,7 @@
package org.apache.flink.api.common.io;
-import java.io.IOException;
-import java.util.ArrayList;
-
import org.apache.flink.annotation.Public;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.apache.flink.api.common.io.statistics.BaseStatistics;
import org.apache.flink.configuration.ConfigConstants;
import org.apache.flink.configuration.Configuration;
@@ -33,7 +28,12 @@ import org.apache.flink.core.fs.FileStatus;
import org.apache.flink.core.fs.FileSystem;
import org.apache.flink.core.fs.Path;
-import com.google.common.base.Charsets;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
/**
* Base implementation for input formats that split the input at a delimiter into records.
@@ -53,6 +53,9 @@ public abstract class DelimitedInputFormat<OT> extends FileInputFormat<OT> {
* The log.
*/
private static final Logger LOG = LoggerFactory.getLogger(DelimitedInputFormat.class);
+
+ /** The default charset to convert strings to bytes */
+ private static final Charset UTF_8_CHARSET = Charset.forName("UTF-8");
/**
* The default read buffer size = 1MB.
@@ -185,7 +188,7 @@ public abstract class DelimitedInputFormat<OT> extends FileInputFormat<OT> {
}
public void setDelimiter(String delimiter) {
- this.delimiter = delimiter.getBytes(Charsets.UTF_8);
+ this.delimiter = delimiter.getBytes(UTF_8_CHARSET);
}
public int getLineLengthLimit() {
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-core/src/main/java/org/apache/flink/api/common/io/FileInputFormat.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/io/FileInputFormat.java b/flink-core/src/main/java/org/apache/flink/api/common/io/FileInputFormat.java
index c4cd2b3..fd69cc3 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/io/FileInputFormat.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/io/FileInputFormat.java
@@ -18,22 +18,10 @@
package org.apache.flink.api.common.io;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import com.google.common.base.Preconditions;
import org.apache.flink.annotation.Public;
import org.apache.flink.api.common.io.compression.DeflateInflaterInputStreamFactory;
import org.apache.flink.api.common.io.compression.GzipInflaterInputStreamFactory;
import org.apache.flink.api.common.io.compression.InflaterInputStreamFactory;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.apache.flink.api.common.io.statistics.BaseStatistics;
import org.apache.flink.configuration.ConfigConstants;
import org.apache.flink.configuration.Configuration;
@@ -45,6 +33,20 @@ import org.apache.flink.core.fs.FileStatus;
import org.apache.flink.core.fs.FileSystem;
import org.apache.flink.core.fs.Path;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
/**
* The base class for {@link RichInputFormat}s that read from files. For specific input types the
* {@link #nextRecord(Object)} and {@link #reachedEnd()} methods need to be implemented.
@@ -143,7 +145,7 @@ public abstract class FileInputFormat<OT> extends RichInputFormat<OT, FileInputS
* @return the extension of the file name or {@code null} if there is no extension.
*/
protected static String extractFileExtension(String fileName) {
- Preconditions.checkNotNull(fileName);
+ checkNotNull(fileName);
int lastPeriodIndex = fileName.lastIndexOf('.');
if (lastPeriodIndex < 0){
return null;
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-core/src/main/java/org/apache/flink/api/common/io/GenericCsvInputFormat.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/io/GenericCsvInputFormat.java b/flink-core/src/main/java/org/apache/flink/api/common/io/GenericCsvInputFormat.java
index cb20f81..31d42ff 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/io/GenericCsvInputFormat.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/io/GenericCsvInputFormat.java
@@ -16,13 +16,8 @@
* limitations under the License.
*/
-
package org.apache.flink.api.common.io;
-import com.google.common.base.Charsets;
-import com.google.common.base.Preconditions;
-import com.google.common.primitives.Ints;
-
import org.apache.flink.annotation.Internal;
import org.apache.flink.core.fs.FileInputSplit;
import org.apache.flink.core.fs.Path;
@@ -30,6 +25,7 @@ import org.apache.flink.types.parser.FieldParser;
import org.apache.flink.types.parser.StringParser;
import org.apache.flink.types.parser.StringValueParser;
import org.apache.flink.util.InstantiationUtil;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -41,13 +37,20 @@ import java.util.ArrayList;
import java.util.Map;
import java.util.TreeMap;
+import static org.apache.flink.util.Preconditions.checkArgument;
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
@Internal
public abstract class GenericCsvInputFormat<OT> extends DelimitedInputFormat<OT> {
- private static final Logger LOG = LoggerFactory.getLogger(GenericCsvInputFormat.class);
-
private static final long serialVersionUID = 1L;
+
+ private static final Logger LOG = LoggerFactory.getLogger(GenericCsvInputFormat.class);
+
+ /** The default charset to convert strings to bytes */
+ private static final Charset UTF_8_CHARSET = Charset.forName("UTF-8");
+
private static final Class<?>[] EMPTY_TYPES = new Class<?>[0];
private static final boolean[] EMPTY_INCLUDED = new boolean[0];
@@ -127,7 +130,7 @@ public abstract class GenericCsvInputFormat<OT> extends DelimitedInputFormat<OT>
}
public void setCommentPrefix(String commentPrefix) {
- setCommentPrefix(commentPrefix, Charsets.UTF_8);
+ setCommentPrefix(commentPrefix, UTF_8_CHARSET);
}
public void setCommentPrefix(String commentPrefix, String charsetName) throws IllegalCharsetNameException, UnsupportedCharsetException {
@@ -171,7 +174,7 @@ public abstract class GenericCsvInputFormat<OT> extends DelimitedInputFormat<OT>
}
public void setFieldDelimiter(String delimiter) {
- this.fieldDelim = delimiter.getBytes(Charsets.UTF_8);
+ this.fieldDelim = delimiter.getBytes(UTF_8_CHARSET);
}
public boolean isLenient() {
@@ -247,9 +250,9 @@ public abstract class GenericCsvInputFormat<OT> extends DelimitedInputFormat<OT>
}
protected void setFieldsGeneric(int[] sourceFieldIndices, Class<?>[] fieldTypes) {
- Preconditions.checkNotNull(sourceFieldIndices);
- Preconditions.checkNotNull(fieldTypes);
- Preconditions.checkArgument(sourceFieldIndices.length == fieldTypes.length,
+ checkNotNull(sourceFieldIndices);
+ checkNotNull(fieldTypes);
+ checkArgument(sourceFieldIndices.length == fieldTypes.length,
"Number of field indices and field types must match.");
for (int i : sourceFieldIndices) {
@@ -258,7 +261,7 @@ public abstract class GenericCsvInputFormat<OT> extends DelimitedInputFormat<OT>
}
}
- int largestFieldIndex = Ints.max(sourceFieldIndices);
+ int largestFieldIndex = max(sourceFieldIndices);
this.fieldIncluded = new boolean[largestFieldIndex + 1];
ArrayList<Class<?>> types = new ArrayList<Class<?>>();
@@ -280,8 +283,8 @@ public abstract class GenericCsvInputFormat<OT> extends DelimitedInputFormat<OT>
}
protected void setFieldsGeneric(boolean[] includedMask, Class<?>[] fieldTypes) {
- Preconditions.checkNotNull(includedMask);
- Preconditions.checkNotNull(fieldTypes);
+ checkNotNull(includedMask);
+ checkNotNull(fieldTypes);
ArrayList<Class<?>> types = new ArrayList<Class<?>>();
@@ -530,4 +533,14 @@ public abstract class GenericCsvInputFormat<OT> extends DelimitedInputFormat<OT>
lastPos = positions[i];
}
}
+
+ private static int max(int[] ints) {
+ checkArgument(ints.length > 0);
+
+ int max = ints[0];
+ for (int i = 1 ; i < ints.length; i++) {
+ max = Math.max(max, ints[i]);
+ }
+ return max;
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-core/src/main/java/org/apache/flink/api/common/operators/GenericDataSinkBase.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/operators/GenericDataSinkBase.java b/flink-core/src/main/java/org/apache/flink/api/common/operators/GenericDataSinkBase.java
index 33f11f3..1f3875d 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/operators/GenericDataSinkBase.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/operators/GenericDataSinkBase.java
@@ -16,7 +16,6 @@
* limitations under the License.
*/
-
package org.apache.flink.api.common.operators;
import java.util.List;
@@ -39,7 +38,7 @@ import org.apache.flink.api.common.typeutils.TypeComparator;
import org.apache.flink.types.Nothing;
import org.apache.flink.util.Visitor;
-import com.google.common.base.Preconditions;
+import static org.apache.flink.util.Preconditions.checkNotNull;
/**
* Operator for nodes that act as data sinks, storing the data they receive.
@@ -66,7 +65,7 @@ public class GenericDataSinkBase<IN> extends Operator<Nothing> {
public GenericDataSinkBase(OutputFormat<IN> f, UnaryOperatorInformation<IN, Nothing> operatorInfo, String name) {
super(operatorInfo, name);
- Preconditions.checkNotNull(f, "The OutputFormat may not be null.");
+ checkNotNull(f, "The OutputFormat may not be null.");
this.formatWrapper = new UserCodeObjectWrapper<OutputFormat<IN>>(f);
}
@@ -79,8 +78,7 @@ public class GenericDataSinkBase<IN> extends Operator<Nothing> {
*/
public GenericDataSinkBase(UserCodeWrapper<? extends OutputFormat<IN>> f, UnaryOperatorInformation<IN, Nothing> operatorInfo, String name) {
super(operatorInfo, name);
- Preconditions.checkNotNull(f, "The OutputFormat class may not be null.");
- this.formatWrapper = f;
+ this.formatWrapper = checkNotNull(f, "The OutputFormat class may not be null.");
}
// --------------------------------------------------------------------------------------------
@@ -100,8 +98,7 @@ public class GenericDataSinkBase<IN> extends Operator<Nothing> {
* @param input The operator to use as the input.
*/
public void setInput(Operator<IN> input) {
- Preconditions.checkNotNull(input, "The input may not be null.");
- this.input = input;
+ this.input = checkNotNull(input, "The input may not be null.");
}
/**
@@ -112,7 +109,7 @@ public class GenericDataSinkBase<IN> extends Operator<Nothing> {
*/
@Deprecated
public void setInputs(Operator<IN>... inputs) {
- Preconditions.checkNotNull(inputs, "The inputs may not be null.");
+ checkNotNull(inputs, "The inputs may not be null.");
this.input = Operator.createUnionCascade(inputs);
}
@@ -124,7 +121,7 @@ public class GenericDataSinkBase<IN> extends Operator<Nothing> {
*/
@Deprecated
public void setInputs(List<Operator<IN>> inputs) {
- Preconditions.checkNotNull(inputs, "The inputs may not be null.");
+ checkNotNull(inputs, "The inputs may not be null.");
this.input = Operator.createUnionCascade(inputs);
}
@@ -136,7 +133,7 @@ public class GenericDataSinkBase<IN> extends Operator<Nothing> {
*/
@Deprecated
public void addInput(Operator<IN>... inputs) {
- Preconditions.checkNotNull(inputs, "The input may not be null.");
+ checkNotNull(inputs, "The input may not be null.");
this.input = Operator.createUnionCascade(this.input, inputs);
}
@@ -149,7 +146,7 @@ public class GenericDataSinkBase<IN> extends Operator<Nothing> {
@SuppressWarnings("unchecked")
@Deprecated
public void addInputs(List<? extends Operator<IN>> inputs) {
- Preconditions.checkNotNull(inputs, "The inputs may not be null.");
+ checkNotNull(inputs, "The inputs may not be null.");
this.input = createUnionCascade(this.input, (Operator<IN>[]) inputs.toArray(new Operator[inputs.size()]));
}
@@ -259,7 +256,7 @@ public class GenericDataSinkBase<IN> extends Operator<Nothing> {
format.configure(this.parameters);
if(format instanceof RichOutputFormat){
- ((RichOutputFormat) format).setRuntimeContext(ctx);
+ ((RichOutputFormat<?>) format).setRuntimeContext(ctx);
}
format.open(0, 1);
for (IN element : inputData) {
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-core/src/main/java/org/apache/flink/api/common/operators/Keys.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/operators/Keys.java b/flink-core/src/main/java/org/apache/flink/api/common/operators/Keys.java
index abe41af..0b771e8 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/operators/Keys.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/operators/Keys.java
@@ -18,10 +18,7 @@
package org.apache.flink.api.common.operators;
-import java.util.ArrayList;
-import java.util.List;
-
-import com.google.common.base.Joiner;
+import org.apache.commons.lang3.StringUtils;
import org.apache.flink.annotation.Internal;
import org.apache.flink.api.common.InvalidProgramException;
@@ -33,7 +30,11 @@ import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.api.java.typeutils.GenericTypeInfo;
import org.apache.flink.api.java.typeutils.TypeExtractor;
-import com.google.common.base.Preconditions;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.flink.util.Preconditions.checkArgument;
+import static org.apache.flink.util.Preconditions.checkNotNull;
@Internal
public abstract class Keys<T> {
@@ -232,7 +233,8 @@ public abstract class Keys<T> {
} else {
rangeCheckFields(keyPositions, type.getArity() - 1);
}
- Preconditions.checkArgument(keyPositions.length > 0, "Grouping fields can not be empty at this point");
+
+ checkArgument(keyPositions.length > 0, "Grouping fields can not be empty at this point");
// extract key field types
CompositeType<T> cType = (CompositeType<T>)type;
@@ -266,7 +268,7 @@ public abstract class Keys<T> {
* Create String-based (nested) field expression keys on a composite type.
*/
public ExpressionKeys(String[] keyExpressions, TypeInformation<T> type) {
- Preconditions.checkNotNull(keyExpressions, "Field expression cannot be null.");
+ checkNotNull(keyExpressions, "Field expression cannot be null.");
this.keyFields = new ArrayList<>(keyExpressions.length);
@@ -375,8 +377,7 @@ public abstract class Keys<T> {
@Override
public String toString() {
- Joiner join = Joiner.on('.');
- return "ExpressionKeys: " + join.join(keyFields);
+ return "ExpressionKeys: " + StringUtils.join(keyFields, '.');
}
public static boolean isSortKey(int fieldPos, TypeInformation<?> type) {
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-core/src/main/java/org/apache/flink/api/common/operators/base/GroupCombineOperatorBase.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/operators/base/GroupCombineOperatorBase.java b/flink-core/src/main/java/org/apache/flink/api/common/operators/base/GroupCombineOperatorBase.java
index b660506..15a0f5a 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/operators/base/GroupCombineOperatorBase.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/operators/base/GroupCombineOperatorBase.java
@@ -18,8 +18,8 @@
package org.apache.flink.api.common.operators.base;
-import com.google.common.base.Preconditions;
import org.apache.commons.lang3.ArrayUtils;
+
import org.apache.flink.annotation.Internal;
import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.InvalidProgramException;
@@ -43,6 +43,8 @@ import java.util.Collections;
import java.util.Comparator;
import java.util.List;
+import static org.apache.flink.util.Preconditions.checkArgument;
+
/**
* Base operator for the combineGroup transformation. It receives the UDF GroupCombineFunction as an input.
* This class is later processed by the compiler to generate the plan.
@@ -109,7 +111,7 @@ public class GroupCombineOperatorBase<IN, OUT, FT extends GroupCombineFunction<I
}
if(sortColumns.length == 0) { // => all reduce. No comparator
- Preconditions.checkArgument(sortOrderings.length == 0);
+ checkArgument(sortOrderings.length == 0);
} else {
final TypeComparator<IN> sortComparator = getTypeComparator(inputType, sortColumns, sortOrderings, executionConfig);
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-core/src/main/java/org/apache/flink/api/common/operators/base/GroupReduceOperatorBase.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/operators/base/GroupReduceOperatorBase.java b/flink-core/src/main/java/org/apache/flink/api/common/operators/base/GroupReduceOperatorBase.java
index 0cc0209..0794a77 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/operators/base/GroupReduceOperatorBase.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/operators/base/GroupReduceOperatorBase.java
@@ -41,13 +41,13 @@ import org.apache.flink.api.common.typeutils.CompositeType;
import org.apache.flink.api.common.typeutils.TypeComparator;
import org.apache.flink.api.common.typeutils.TypeSerializer;
-import com.google.common.base.Preconditions;
-
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
+import static org.apache.flink.util.Preconditions.checkArgument;
+
/**
* @see org.apache.flink.api.common.functions.GroupReduceFunction
*/
@@ -181,7 +181,7 @@ public class GroupReduceOperatorBase<IN, OUT, FT extends GroupReduceFunction<IN,
}
if(sortColumns.length == 0) { // => all reduce. No comparator
- Preconditions.checkArgument(sortOrderings.length == 0);
+ checkArgument(sortOrderings.length == 0);
} else {
final TypeComparator<IN> sortComparator = getTypeComparator(inputType, sortColumns, sortOrderings, executionConfig);
Collections.sort(inputData, new Comparator<IN>() {
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-core/src/main/java/org/apache/flink/api/common/operators/util/FieldList.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/operators/util/FieldList.java b/flink-core/src/main/java/org/apache/flink/api/common/operators/util/FieldList.java
index c69c875..15a993c 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/operators/util/FieldList.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/operators/util/FieldList.java
@@ -16,16 +16,16 @@
* limitations under the License.
*/
-
package org.apache.flink.api.common.operators.util;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
-import com.google.common.base.Preconditions;
import org.apache.flink.annotation.Internal;
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
/**
* Immutable ordered list of fields IDs.
*/
@@ -44,7 +44,7 @@ public class FieldList extends FieldSet {
}
public FieldList(Integer fieldId) {
- super(Collections.singletonList(Preconditions.checkNotNull(fieldId, "The fields ID must not be null.")));
+ super(Collections.singletonList(checkNotNull(fieldId, "The fields ID must not be null.")));
}
public FieldList(int... columnIndexes) {
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-core/src/main/java/org/apache/flink/api/common/operators/util/UserCodeObjectWrapper.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/operators/util/UserCodeObjectWrapper.java b/flink-core/src/main/java/org/apache/flink/api/common/operators/util/UserCodeObjectWrapper.java
index 890caac..5d75c95 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/operators/util/UserCodeObjectWrapper.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/operators/util/UserCodeObjectWrapper.java
@@ -27,7 +27,8 @@ import java.lang.reflect.Modifier;
import org.apache.flink.annotation.Internal;
import org.apache.flink.api.common.NonSerializableUserCodeException;
-import com.google.common.base.Preconditions;
+import static org.apache.flink.util.Preconditions.checkArgument;
+import static org.apache.flink.util.Preconditions.checkNotNull;
/**
* This holds an actual object containing user defined code.
@@ -39,8 +40,8 @@ public class UserCodeObjectWrapper<T> implements UserCodeWrapper<T> {
private final T userCodeObject;
public UserCodeObjectWrapper(T userCodeObject) {
- Preconditions.checkNotNull(userCodeObject, "The user code object may not be null.");
- Preconditions.checkArgument(userCodeObject instanceof Serializable, "User code object is not serializable: " + userCodeObject.getClass().getName());
+ checkNotNull(userCodeObject, "The user code object may not be null.");
+ checkArgument(userCodeObject instanceof Serializable, "User code object is not serializable: " + userCodeObject.getClass().getName());
this.userCodeObject = userCodeObject;
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/BasicArrayTypeInfo.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/BasicArrayTypeInfo.java b/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/BasicArrayTypeInfo.java
index 2c61fb2..25b2850 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/BasicArrayTypeInfo.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/BasicArrayTypeInfo.java
@@ -22,7 +22,6 @@ import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
-import com.google.common.base.Preconditions;
import org.apache.flink.annotation.PublicEvolving;
import org.apache.flink.annotation.Public;
import org.apache.flink.api.common.ExecutionConfig;
@@ -31,6 +30,8 @@ import org.apache.flink.api.common.typeutils.base.array.StringArraySerializer;
import org.apache.flink.api.common.functions.InvalidTypesException;
import org.apache.flink.api.common.typeutils.base.GenericArraySerializer;
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
@Public
public final class BasicArrayTypeInfo<T, C> extends TypeInformation<T> {
@@ -51,11 +52,10 @@ public final class BasicArrayTypeInfo<T, C> extends TypeInformation<T> {
private final Class<T> arrayClass;
private final TypeInformation<C> componentInfo;
-
- @SuppressWarnings("unchecked")
+
private BasicArrayTypeInfo(Class<T> arrayClass, BasicTypeInfo<C> componentInfo) {
- this.arrayClass = Preconditions.checkNotNull(arrayClass);
- this.componentInfo = Preconditions.checkNotNull(componentInfo);
+ this.arrayClass = checkNotNull(arrayClass);
+ this.componentInfo = checkNotNull(componentInfo);
}
// --------------------------------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/BasicTypeInfo.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/BasicTypeInfo.java b/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/BasicTypeInfo.java
index 4eb70c1..c1d5605 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/BasicTypeInfo.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/BasicTypeInfo.java
@@ -25,7 +25,6 @@ import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
-import com.google.common.base.Preconditions;
import org.apache.flink.annotation.PublicEvolving;
import org.apache.flink.annotation.Public;
import org.apache.flink.api.common.ExecutionConfig;
@@ -54,6 +53,7 @@ import org.apache.flink.api.common.typeutils.base.StringComparator;
import org.apache.flink.api.common.typeutils.base.StringSerializer;
import org.apache.flink.api.common.typeutils.base.VoidSerializer;
+import static org.apache.flink.util.Preconditions.checkNotNull;
/**
* Type information for primitive types (int, long, double, byte, ...), String, Date, and Void.
@@ -87,9 +87,9 @@ public class BasicTypeInfo<T> extends TypeInformation<T> implements AtomicType<T
protected BasicTypeInfo(Class<T> clazz, Class<?>[] possibleCastTargetTypes, TypeSerializer<T> serializer, Class<? extends TypeComparator<T>> comparatorClass) {
- this.clazz = Preconditions.checkNotNull(clazz);
- this.possibleCastTargetTypes = Preconditions.checkNotNull(possibleCastTargetTypes);
- this.serializer = Preconditions.checkNotNull(serializer);
+ this.clazz = checkNotNull(clazz);
+ this.possibleCastTargetTypes = checkNotNull(possibleCastTargetTypes);
+ this.serializer = checkNotNull(serializer);
// comparator can be null as in VOID_TYPE_INFO
this.comparatorClass = comparatorClass;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/FractionalTypeInfo.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/FractionalTypeInfo.java b/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/FractionalTypeInfo.java
index aa22ac6..b4e53d4 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/FractionalTypeInfo.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/FractionalTypeInfo.java
@@ -18,13 +18,14 @@
package org.apache.flink.api.common.typeinfo;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Sets;
import org.apache.flink.annotation.Public;
import org.apache.flink.api.common.typeutils.TypeComparator;
import org.apache.flink.api.common.typeutils.TypeSerializer;
-import java.util.Set;
+import java.util.Arrays;
+import java.util.HashSet;
+
+import static org.apache.flink.util.Preconditions.checkArgument;
/**
* Type information for numeric fractional primitive types (double, float).
@@ -34,15 +35,15 @@ public class FractionalTypeInfo<T> extends NumericTypeInfo<T> {
private static final long serialVersionUID = 554334260950199994L;
- private static final Set<Class<?>> fractionalTypes = Sets.<Class<?>>newHashSet(
- Double.class,
- Float.class
- );
+ private static final HashSet<Class<?>> fractionalTypes = new HashSet<Class<?>>(
+ Arrays.asList(
+ Double.class,
+ Float.class));
protected FractionalTypeInfo(Class<T> clazz, Class<?>[] possibleCastTargetTypes, TypeSerializer<T> serializer, Class<? extends TypeComparator<T>> comparatorClass) {
super(clazz, possibleCastTargetTypes, serializer, comparatorClass);
- Preconditions.checkArgument(fractionalTypes.contains(clazz), "The given class " +
- clazz.getSimpleName() + " is not a fractional type.");
+ checkArgument(fractionalTypes.contains(clazz),
+ "The given class %s is not a fractional type.", clazz.getSimpleName());
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/IntegerTypeInfo.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/IntegerTypeInfo.java b/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/IntegerTypeInfo.java
index bff3ab7..02b416a 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/IntegerTypeInfo.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/IntegerTypeInfo.java
@@ -18,13 +18,14 @@
package org.apache.flink.api.common.typeinfo;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Sets;
import org.apache.flink.annotation.Public;
import org.apache.flink.api.common.typeutils.TypeComparator;
import org.apache.flink.api.common.typeutils.TypeSerializer;
-import java.util.Set;
+import java.util.Arrays;
+import java.util.HashSet;
+
+import static org.apache.flink.util.Preconditions.checkArgument;
/**
* Type information for numeric integer primitive types: int, long, byte, short, character.
@@ -34,18 +35,18 @@ public class IntegerTypeInfo<T> extends NumericTypeInfo<T> {
private static final long serialVersionUID = -8068827354966766955L;
- private static final Set<Class<?>> integerTypes = Sets.<Class<?>>newHashSet(
- Integer.class,
- Long.class,
- Byte.class,
- Short.class,
- Character.class
- );
+ private static final HashSet<Class<?>> integerTypes = new HashSet<Class<?>>(
+ Arrays.asList(
+ Integer.class,
+ Long.class,
+ Byte.class,
+ Short.class,
+ Character.class));
protected IntegerTypeInfo(Class<T> clazz, Class<?>[] possibleCastTargetTypes, TypeSerializer<T> serializer, Class<? extends TypeComparator<T>> comparatorClass) {
super(clazz, possibleCastTargetTypes, serializer, comparatorClass);
- Preconditions.checkArgument(integerTypes.contains(clazz), "The given class " +
- clazz.getSimpleName() + " is not a integer type.");
+ checkArgument(integerTypes.contains(clazz),
+ "The given class %s is not a integer type.", clazz.getSimpleName());
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/NumericTypeInfo.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/NumericTypeInfo.java b/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/NumericTypeInfo.java
index 6969520..fea8abe 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/NumericTypeInfo.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/NumericTypeInfo.java
@@ -18,13 +18,14 @@
package org.apache.flink.api.common.typeinfo;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Sets;
import org.apache.flink.annotation.Public;
import org.apache.flink.api.common.typeutils.TypeComparator;
import org.apache.flink.api.common.typeutils.TypeSerializer;
-import java.util.Set;
+import java.util.Arrays;
+import java.util.HashSet;
+
+import static org.apache.flink.util.Preconditions.checkArgument;
/**
* Type information for numeric primitive types: int, long, double, byte, short, float, char.
@@ -34,21 +35,21 @@ public abstract class NumericTypeInfo<T> extends BasicTypeInfo<T> {
private static final long serialVersionUID = -5937777910658986986L;
- private static final Set<Class<?>> numericalTypes = Sets.<Class<?>>newHashSet(
- Integer.class,
- Long.class,
- Double.class,
- Byte.class,
- Short.class,
- Float.class,
- Character.class
- );
+ private static final HashSet<Class<?>> numericalTypes = new HashSet<Class<?>>(
+ Arrays.asList(
+ Integer.class,
+ Long.class,
+ Double.class,
+ Byte.class,
+ Short.class,
+ Float.class,
+ Character.class));
protected NumericTypeInfo(Class<T> clazz, Class<?>[] possibleCastTargetTypes, TypeSerializer<T> serializer, Class<? extends
TypeComparator<T>> comparatorClass) {
super(clazz, possibleCastTargetTypes, serializer, comparatorClass);
- Preconditions.checkArgument(numericalTypes.contains(clazz), "The given class " +
- clazz.getSimpleName() + " is not a numerical type.");
+ checkArgument(numericalTypes.contains(clazz),
+ "The given class %s is not a numerical type", clazz.getSimpleName());
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/PrimitiveArrayTypeInfo.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/PrimitiveArrayTypeInfo.java b/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/PrimitiveArrayTypeInfo.java
index 2c75458..1c6ce00 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/PrimitiveArrayTypeInfo.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/PrimitiveArrayTypeInfo.java
@@ -18,13 +18,8 @@
package org.apache.flink.api.common.typeinfo;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Objects;
-
-import com.google.common.base.Preconditions;
-import org.apache.flink.annotation.PublicEvolving;
import org.apache.flink.annotation.Public;
+import org.apache.flink.annotation.PublicEvolving;
import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.functions.InvalidTypesException;
import org.apache.flink.api.common.typeutils.TypeSerializer;
@@ -46,6 +41,13 @@ import org.apache.flink.api.common.typeutils.base.array.PrimitiveArrayComparator
import org.apache.flink.api.common.typeutils.base.array.ShortPrimitiveArrayComparator;
import org.apache.flink.api.common.typeutils.base.array.ShortPrimitiveArraySerializer;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.apache.flink.util.Preconditions.checkArgument;
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
/**
* A {@link TypeInformation} for arrays of primitive types (int, long, double, ...).
* Supports the creation of dedicated efficient serializers for these types.
@@ -83,11 +85,11 @@ public class PrimitiveArrayTypeInfo<T> extends TypeInformation<T> implements Ato
* @param comparatorClass The class of the array comparator
*/
private PrimitiveArrayTypeInfo(Class<T> arrayClass, TypeSerializer<T> serializer, Class<? extends PrimitiveArrayComparator<T, ?>> comparatorClass) {
- this.arrayClass = Preconditions.checkNotNull(arrayClass);
- this.serializer = Preconditions.checkNotNull(serializer);
- this.comparatorClass = Preconditions.checkNotNull(comparatorClass);
+ this.arrayClass = checkNotNull(arrayClass);
+ this.serializer = checkNotNull(serializer);
+ this.comparatorClass = checkNotNull(comparatorClass);
- Preconditions.checkArgument(
+ checkArgument(
arrayClass.isArray() && arrayClass.getComponentType().isPrimitive(),
"Class must represent an array of primitives");
}
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompositeType.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompositeType.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompositeType.java
index 19b6eaf..4bf17ea 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompositeType.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompositeType.java
@@ -22,13 +22,13 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
-import com.google.common.base.Preconditions;
import org.apache.flink.annotation.PublicEvolving;
import org.apache.flink.annotation.Public;
import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.typeinfo.AtomicType;
import org.apache.flink.api.common.typeinfo.TypeInformation;
+import static org.apache.flink.util.Preconditions.checkNotNull;
/**
* Base type information class for Tuple and Pojo types
@@ -44,7 +44,7 @@ public abstract class CompositeType<T> extends TypeInformation<T> {
@PublicEvolving
public CompositeType(Class<T> typeClass) {
- this.typeClass = Preconditions.checkNotNull(typeClass);
+ this.typeClass = checkNotNull(typeClass);
}
/**
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/EnumSerializer.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/EnumSerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/EnumSerializer.java
index c64e399..21a6ea0 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/EnumSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/EnumSerializer.java
@@ -22,12 +22,13 @@ import java.io.IOException;
import java.io.ObjectInputStream;
import java.lang.reflect.Method;
-import com.google.common.base.Preconditions;
import org.apache.flink.annotation.Internal;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.core.memory.DataInputView;
import org.apache.flink.core.memory.DataOutputView;
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
@Internal
public final class EnumSerializer<T extends Enum<T>> extends TypeSerializer<T> {
@@ -38,7 +39,7 @@ public final class EnumSerializer<T extends Enum<T>> extends TypeSerializer<T> {
private final Class<T> enumClass;
public EnumSerializer(Class<T> enumClass) {
- this.enumClass = Preconditions.checkNotNull(enumClass);
+ this.enumClass = checkNotNull(enumClass);
this.values = createValues(enumClass);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/GenericArraySerializer.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/GenericArraySerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/GenericArraySerializer.java
index eb6e708..f35d71b 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/GenericArraySerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/GenericArraySerializer.java
@@ -21,12 +21,13 @@ package org.apache.flink.api.common.typeutils.base;
import java.io.IOException;
import java.lang.reflect.Array;
-import com.google.common.base.Preconditions;
import org.apache.flink.annotation.Internal;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.core.memory.DataInputView;
import org.apache.flink.core.memory.DataOutputView;
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
/**
* A serializer for arrays of objects.
*
@@ -45,8 +46,8 @@ public final class GenericArraySerializer<C> extends TypeSerializer<C[]> {
public GenericArraySerializer(Class<C> componentClass, TypeSerializer<C> componentSerializer) {
- this.componentClass = Preconditions.checkNotNull(componentClass);
- this.componentSerializer = Preconditions.checkNotNull(componentSerializer);
+ this.componentClass = checkNotNull(componentClass);
+ this.componentSerializer = checkNotNull(componentSerializer);
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-core/src/main/java/org/apache/flink/api/java/typeutils/EnumTypeInfo.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/EnumTypeInfo.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/EnumTypeInfo.java
index aec3c1d..61d9ae4 100644
--- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/EnumTypeInfo.java
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/EnumTypeInfo.java
@@ -18,7 +18,7 @@
package org.apache.flink.api.java.typeutils;
-import com.google.common.base.Preconditions;
+import org.apache.flink.annotation.Public;
import org.apache.flink.annotation.PublicEvolving;
import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.typeinfo.AtomicType;
@@ -27,7 +27,8 @@ import org.apache.flink.api.common.typeutils.TypeComparator;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.common.typeutils.base.EnumComparator;
import org.apache.flink.api.common.typeutils.base.EnumSerializer;
-import org.apache.flink.annotation.Public;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
/**
* A {@link TypeInformation} for java enumeration types.
@@ -43,7 +44,7 @@ public class EnumTypeInfo<T extends Enum<T>> extends TypeInformation<T> implemen
@PublicEvolving
public EnumTypeInfo(Class<T> typeClass) {
- Preconditions.checkNotNull(typeClass, "Enum type class must not be null.");
+ checkNotNull(typeClass, "Enum type class must not be null.");
if (!Enum.class.isAssignableFrom(typeClass) ) {
throw new IllegalArgumentException("EnumTypeInfo can only be used for subclasses of " + Enum.class.getName());
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-core/src/main/java/org/apache/flink/api/java/typeutils/GenericTypeInfo.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/GenericTypeInfo.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/GenericTypeInfo.java
index 0cca8bd..bc4e87a 100644
--- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/GenericTypeInfo.java
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/GenericTypeInfo.java
@@ -18,7 +18,6 @@
package org.apache.flink.api.java.typeutils;
-import com.google.common.base.Preconditions;
import org.apache.flink.annotation.PublicEvolving;
import org.apache.flink.annotation.Public;
import org.apache.flink.api.common.ExecutionConfig;
@@ -29,6 +28,7 @@ import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.java.typeutils.runtime.GenericTypeComparator;
import org.apache.flink.api.java.typeutils.runtime.kryo.KryoSerializer;
+import static org.apache.flink.util.Preconditions.checkNotNull;
@Public
public class GenericTypeInfo<T> extends TypeInformation<T> implements AtomicType<T> {
@@ -39,7 +39,7 @@ public class GenericTypeInfo<T> extends TypeInformation<T> implements AtomicType
@PublicEvolving
public GenericTypeInfo(Class<T> typeClass) {
- this.typeClass = Preconditions.checkNotNull(typeClass);
+ this.typeClass = checkNotNull(typeClass);
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-core/src/main/java/org/apache/flink/api/java/typeutils/ObjectArrayTypeInfo.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/ObjectArrayTypeInfo.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/ObjectArrayTypeInfo.java
index 1e8fbe2..2edb3b9 100644
--- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/ObjectArrayTypeInfo.java
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/ObjectArrayTypeInfo.java
@@ -27,7 +27,8 @@ import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.common.typeutils.base.GenericArraySerializer;
-import com.google.common.base.Preconditions;
+import static org.apache.flink.util.Preconditions.checkNotNull;
+import static org.apache.flink.util.Preconditions.checkArgument;
@Public
public class ObjectArrayTypeInfo<T, C> extends TypeInformation<T> {
@@ -38,8 +39,8 @@ public class ObjectArrayTypeInfo<T, C> extends TypeInformation<T> {
private final TypeInformation<C> componentInfo;
private ObjectArrayTypeInfo(Class<T> arrayType, TypeInformation<C> componentInfo) {
- this.arrayType = Preconditions.checkNotNull(arrayType);
- this.componentInfo = Preconditions.checkNotNull(componentInfo);
+ this.arrayType = checkNotNull(arrayType);
+ this.componentInfo = checkNotNull(componentInfo);
}
// --------------------------------------------------------------------------------------------
@@ -128,9 +129,9 @@ public class ObjectArrayTypeInfo<T, C> extends TypeInformation<T> {
@PublicEvolving
public static <T, C> ObjectArrayTypeInfo<T, C> getInfoFor(Class<T> arrayClass, TypeInformation<C> componentInfo) {
- Preconditions.checkNotNull(arrayClass);
- Preconditions.checkNotNull(componentInfo);
- Preconditions.checkArgument(arrayClass.isArray(), "Class " + arrayClass + " must be an array.");
+ checkNotNull(arrayClass);
+ checkNotNull(componentInfo);
+ checkArgument(arrayClass.isArray(), "Class " + arrayClass + " must be an array.");
return new ObjectArrayTypeInfo<T, C>(arrayClass, componentInfo);
}
@@ -146,7 +147,7 @@ public class ObjectArrayTypeInfo<T, C> extends TypeInformation<T> {
@SuppressWarnings("unchecked")
@PublicEvolving
public static <T, C> ObjectArrayTypeInfo<T, C> getInfoFor(TypeInformation<C> componentInfo) {
- Preconditions.checkNotNull(componentInfo);
+ checkNotNull(componentInfo);
return new ObjectArrayTypeInfo<T, C>(
(Class<T>)Array.newInstance(componentInfo.getTypeClass(), 0).getClass(),
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-core/src/main/java/org/apache/flink/api/java/typeutils/PojoField.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/PojoField.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/PojoField.java
index c37fc77..026cfa6 100644
--- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/PojoField.java
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/PojoField.java
@@ -25,10 +25,11 @@ import java.io.Serializable;
import java.lang.reflect.Field;
import java.util.Objects;
-import com.google.common.base.Preconditions;
import org.apache.flink.annotation.Internal;
import org.apache.flink.api.common.typeinfo.TypeInformation;
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
/**
* Represent a field definition for {@link PojoTypeInfo} type of objects.
*/
@@ -41,8 +42,8 @@ public class PojoField implements Serializable {
private final TypeInformation<?> type;
public PojoField(Field field, TypeInformation<?> type) {
- this.field = Preconditions.checkNotNull(field);
- this.type = Preconditions.checkNotNull(type);
+ this.field = checkNotNull(field);
+ this.type = checkNotNull(type);
}
public Field getField() {
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-core/src/main/java/org/apache/flink/api/java/typeutils/PojoTypeInfo.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/PojoTypeInfo.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/PojoTypeInfo.java
index be2a027..9c65263 100644
--- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/PojoTypeInfo.java
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/PojoTypeInfo.java
@@ -18,19 +18,9 @@
package org.apache.flink.api.java.typeutils;
-import java.lang.reflect.Field;
-import java.lang.reflect.Modifier;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Comparator;
-import java.util.List;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import com.google.common.base.Preconditions;
-
-import org.apache.flink.annotation.PublicEvolving;
+import org.apache.commons.lang3.StringUtils;
import org.apache.flink.annotation.Public;
+import org.apache.flink.annotation.PublicEvolving;
import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.operators.Keys.ExpressionKeys;
import org.apache.flink.api.common.typeinfo.TypeInformation;
@@ -40,10 +30,20 @@ import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.java.typeutils.runtime.AvroSerializer;
import org.apache.flink.api.java.typeutils.runtime.PojoComparator;
import org.apache.flink.api.java.typeutils.runtime.PojoSerializer;
-
-import com.google.common.base.Joiner;
import org.apache.flink.api.java.typeutils.runtime.kryo.KryoSerializer;
+import java.lang.reflect.Field;
+import java.lang.reflect.Modifier;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import static org.apache.flink.util.Preconditions.checkArgument;
+import static org.apache.flink.util.Preconditions.checkState;
+
/**
* TypeInformation for "Java Beans"-style types. Flink refers to them as POJOs,
* since the conditions are slightly different from Java Beans.
@@ -78,8 +78,8 @@ public class PojoTypeInfo<T> extends CompositeType<T> {
public PojoTypeInfo(Class<T> typeClass, List<PojoField> fields) {
super(typeClass);
- Preconditions.checkArgument(Modifier.isPublic(typeClass.getModifiers()),
- "POJO " + typeClass + " is not public");
+ checkArgument(Modifier.isPublic(typeClass.getModifiers()),
+ "POJO %s is not public", typeClass);
this.fields = fields.toArray(new PojoField[fields.size()]);
@@ -350,7 +350,7 @@ public class PojoTypeInfo<T> extends CompositeType<T> {
fieldStrings.add(field.getField().getName() + ": " + field.getTypeInformation().toString());
}
return "PojoType<" + getTypeClass().getName()
- + ", fields = [" + Joiner.on(", ").join(fieldStrings) + "]"
+ + ", fields = [" + StringUtils.join(fieldStrings, ", ") + "]"
+ ">";
}
@@ -381,15 +381,15 @@ public class PojoTypeInfo<T> extends CompositeType<T> {
@Override
public TypeComparator<T> createTypeComparator(ExecutionConfig config) {
- Preconditions.checkState(
+ checkState(
keyFields.size() > 0,
"No keys were defined for the PojoTypeComparatorBuilder.");
- Preconditions.checkState(
+ checkState(
fieldComparators.size() > 0,
"No type comparators were defined for the PojoTypeComparatorBuilder.");
- Preconditions.checkState(
+ checkState(
keyFields.size() == fieldComparators.size(),
"Number of key fields and field comparators is not equal.");
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-core/src/main/java/org/apache/flink/api/java/typeutils/TupleTypeInfo.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/TupleTypeInfo.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/TupleTypeInfo.java
index 9ecbe73..051ad0df 100644
--- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/TupleTypeInfo.java
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/TupleTypeInfo.java
@@ -22,24 +22,26 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
-import com.google.common.base.Preconditions;
-import com.google.common.primitives.Ints;
import org.apache.flink.annotation.PublicEvolving;
import org.apache.flink.annotation.Public;
import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.functions.InvalidTypesException;
import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
import org.apache.flink.api.common.typeinfo.TypeInformation;
-//CHECKSTYLE.OFF: AvoidStarImport - Needed for TupleGenerator
import org.apache.flink.api.common.typeutils.TypeComparator;
import org.apache.flink.api.common.typeutils.TypeSerializer;
-import org.apache.flink.api.java.tuple.*;
import org.apache.flink.api.java.typeutils.runtime.Tuple0Serializer;
-//CHECKSTYLE.ON: AvoidStarImport
import org.apache.flink.api.java.typeutils.runtime.TupleComparator;
import org.apache.flink.api.java.typeutils.runtime.TupleSerializer;
import org.apache.flink.types.Value;
+//CHECKSTYLE.OFF: AvoidStarImport - Needed for TupleGenerator
+import org.apache.flink.api.java.tuple.*;
+//CHECKSTYLE.ON: AvoidStarImport
+
+import static org.apache.flink.util.Preconditions.checkArgument;
+import static org.apache.flink.util.Preconditions.checkState;
+
/**
* A {@link TypeInformation} for the tuple types of the Java API.
*
@@ -62,7 +64,7 @@ public final class TupleTypeInfo<T extends Tuple> extends TupleTypeInfoBase<T> {
public TupleTypeInfo(Class<T> tupleType, TypeInformation<?>... types) {
super(tupleType, types);
- Preconditions.checkArgument(
+ checkArgument(
types.length <= Tuple.MAX_ARITY,
"The tuple type exceeds the maximum supported arity.");
@@ -131,24 +133,24 @@ public final class TupleTypeInfo<T extends Tuple> extends TupleTypeInfoBase<T> {
@Override
public TypeComparator<T> createTypeComparator(ExecutionConfig config) {
- Preconditions.checkState(
+ checkState(
fieldComparators.size() > 0,
"No field comparators were defined for the TupleTypeComparatorBuilder."
);
- Preconditions.checkState(
+ checkState(
logicalKeyFields.size() > 0,
"No key fields were defined for the TupleTypeComparatorBuilder."
);
- Preconditions.checkState(
+ checkState(
fieldComparators.size() == logicalKeyFields.size(),
"The number of field comparators and key fields is not equal."
);
final int maxKey = Collections.max(logicalKeyFields);
- Preconditions.checkState(
+ checkState(
maxKey >= 0,
"The maximum key field must be greater or equal than 0."
);
@@ -160,7 +162,7 @@ public final class TupleTypeInfo<T extends Tuple> extends TupleTypeInfoBase<T> {
}
return new TupleComparator<T>(
- Ints.toArray(logicalKeyFields),
+ listToPrimitives(logicalKeyFields),
fieldComparators.toArray(new TypeComparator[fieldComparators.size()]),
fieldSerializers
);
@@ -255,4 +257,12 @@ public final class TupleTypeInfo<T extends Tuple> extends TupleTypeInfoBase<T> {
return (TupleTypeInfo<X>) new TupleTypeInfo<>(infos);
}
+
+ private static int[] listToPrimitives(ArrayList<Integer> ints) {
+ int[] result = new int[ints.size()];
+ for (int i = 0; i < result.length; i++) {
+ result[i] = ints.get(i);
+ }
+ return result;
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-core/src/main/java/org/apache/flink/api/java/typeutils/TupleTypeInfoBase.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/TupleTypeInfoBase.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/TupleTypeInfoBase.java
index 753eb66..807fd54 100644
--- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/TupleTypeInfoBase.java
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/TupleTypeInfoBase.java
@@ -23,12 +23,12 @@ import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import com.google.common.base.Preconditions;
-
import org.apache.flink.api.common.operators.Keys.ExpressionKeys;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.common.typeutils.CompositeType;
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
public abstract class TupleTypeInfoBase<T> extends CompositeType<T> {
private static final long serialVersionUID = 1L;
@@ -52,7 +52,7 @@ public abstract class TupleTypeInfoBase<T> extends CompositeType<T> {
public TupleTypeInfoBase(Class<T> tupleType, TypeInformation<?>... types) {
super(tupleType);
- this.types = Preconditions.checkNotNull(types);
+ this.types = checkNotNull(types);
int fieldCounter = 0;
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-core/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java
index 151f359..0469cc2 100644
--- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java
@@ -66,7 +66,7 @@ import org.apache.hadoop.io.Writable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.base.Preconditions;
+import static org.apache.flink.util.Preconditions.checkNotNull;
/**
* A utility for reflection analysis on classes, to determine the return type of implementations of transformation
@@ -902,7 +902,7 @@ public class TypeExtractor {
if (isClassType(originalType)) {
originalTypeAsClass = typeToClass(originalType);
}
- Preconditions.checkNotNull(originalTypeAsClass, "originalType has an unexpected type");
+ checkNotNull(originalTypeAsClass, "originalType has an unexpected type");
// check if the class we assumed to conform to the defining type so far is actually a pojo because the
// original type contains additional fields.
// check for additional fields.
@@ -1466,7 +1466,7 @@ public class TypeExtractor {
@SuppressWarnings({ "unchecked", "rawtypes" })
private <OUT,IN1,IN2> TypeInformation<OUT> privateGetForClass(Class<OUT> clazz, ArrayList<Type> typeHierarchy,
ParameterizedType parameterizedType, TypeInformation<IN1> in1Type, TypeInformation<IN2> in2Type) {
- Preconditions.checkNotNull(clazz);
+ checkNotNull(clazz);
// Object is handled as generic type info
if (clazz.equals(Object.class)) {
@@ -1822,7 +1822,7 @@ public class TypeExtractor {
@SuppressWarnings({ "unchecked", "rawtypes" })
private <X> TypeInformation<X> privateGetForObject(X value) {
- Preconditions.checkNotNull(value);
+ checkNotNull(value);
// check if we can extract the types from tuples, otherwise work with the class
if (value instanceof Tuple) {
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-core/src/main/java/org/apache/flink/api/java/typeutils/ValueTypeInfo.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/ValueTypeInfo.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/ValueTypeInfo.java
index 7c173c0..495a324 100644
--- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/ValueTypeInfo.java
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/ValueTypeInfo.java
@@ -18,9 +18,8 @@
package org.apache.flink.api.java.typeutils;
-import com.google.common.base.Preconditions;
-import org.apache.flink.annotation.PublicEvolving;
import org.apache.flink.annotation.Public;
+import org.apache.flink.annotation.PublicEvolving;
import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.functions.InvalidTypesException;
import org.apache.flink.api.common.typeinfo.AtomicType;
@@ -44,6 +43,9 @@ import org.apache.flink.types.ShortValue;
import org.apache.flink.types.StringValue;
import org.apache.flink.types.Value;
+import static org.apache.flink.util.Preconditions.checkArgument;
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
/**
* Type information for data types that extend the {@link Value} interface. The value
* interface allows types to define their custom serialization and deserialization routines.
@@ -70,11 +72,11 @@ public class ValueTypeInfo<T extends Value> extends TypeInformation<T> implement
@PublicEvolving
public ValueTypeInfo(Class<T> type) {
- this.type = Preconditions.checkNotNull(type);
+ this.type = checkNotNull(type);
- Preconditions.checkArgument(
+ checkArgument(
Value.class.isAssignableFrom(type) || type.equals(Value.class),
- "ValueTypeInfo can only be used for subclasses of " + Value.class.getName());
+ "ValueTypeInfo can only be used for subclasses of %s", Value.class.getName());
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-core/src/main/java/org/apache/flink/api/java/typeutils/WritableTypeInfo.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/WritableTypeInfo.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/WritableTypeInfo.java
index 5e3b2bc..7ca7a91 100644
--- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/WritableTypeInfo.java
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/WritableTypeInfo.java
@@ -18,7 +18,6 @@
package org.apache.flink.api.java.typeutils;
-import com.google.common.base.Preconditions;
import org.apache.flink.annotation.PublicEvolving;
import org.apache.flink.annotation.Public;
import org.apache.flink.api.common.ExecutionConfig;
@@ -29,8 +28,12 @@ import org.apache.flink.api.common.typeutils.TypeComparator;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.java.typeutils.runtime.WritableComparator;
import org.apache.flink.api.java.typeutils.runtime.WritableSerializer;
+
import org.apache.hadoop.io.Writable;
+import static org.apache.flink.util.Preconditions.checkNotNull;
+import static org.apache.flink.util.Preconditions.checkArgument;
+
/**
* Type information for data types that extend Hadoop's {@link Writable} interface. The Writable
* interface defines the serialization and deserialization routines for the data type.
@@ -46,11 +49,11 @@ public class WritableTypeInfo<T extends Writable> extends TypeInformation<T> imp
@PublicEvolving
public WritableTypeInfo(Class<T> typeClass) {
- this.typeClass = Preconditions.checkNotNull(typeClass);
+ this.typeClass = checkNotNull(typeClass);
- Preconditions.checkArgument(
+ checkArgument(
Writable.class.isAssignableFrom(typeClass) && !typeClass.equals(Writable.class),
- "WritableTypeInfo can only be used for subclasses of " + Writable.class.getName());
+ "WritableTypeInfo can only be used for subclasses of %s", Writable.class.getName());
}
@SuppressWarnings({ "rawtypes", "unchecked" })
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/AvroSerializer.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/AvroSerializer.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/AvroSerializer.java
index bc04367..4c2a7f9 100644
--- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/AvroSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/AvroSerializer.java
@@ -20,11 +20,11 @@ package org.apache.flink.api.java.typeutils.runtime;
import java.io.IOException;
-import com.google.common.base.Preconditions;
import org.apache.avro.generic.GenericData;
import org.apache.avro.reflect.ReflectDatumReader;
import org.apache.avro.reflect.ReflectDatumWriter;
import org.apache.avro.util.Utf8;
+
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.java.typeutils.runtime.kryo.Serializers;
import org.apache.flink.core.memory.DataInputView;
@@ -34,6 +34,7 @@ import org.apache.flink.util.InstantiationUtil;
import com.esotericsoftware.kryo.Kryo;
import org.objenesis.strategy.StdInstantiatorStrategy;
+import static org.apache.flink.util.Preconditions.checkNotNull;
/**
* General purpose serialization. Currently using Apache Avro's Reflect-serializers for serialization and
@@ -66,8 +67,8 @@ public final class AvroSerializer<T> extends TypeSerializer<T> {
}
public AvroSerializer(Class<T> type, Class<? extends T> typeToInstantiate) {
- this.type = Preconditions.checkNotNull(type);
- this.typeToInstantiate = Preconditions.checkNotNull(typeToInstantiate);
+ this.type = checkNotNull(type);
+ this.typeToInstantiate = checkNotNull(typeToInstantiate);
InstantiationUtil.checkForInstantiation(typeToInstantiate);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/CopyableValueSerializer.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/CopyableValueSerializer.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/CopyableValueSerializer.java
index 9e46f27..f30a767 100644
--- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/CopyableValueSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/CopyableValueSerializer.java
@@ -20,13 +20,13 @@ package org.apache.flink.api.java.typeutils.runtime;
import java.io.IOException;
-import com.google.common.base.Preconditions;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.core.memory.DataInputView;
import org.apache.flink.core.memory.DataOutputView;
import org.apache.flink.types.CopyableValue;
import org.apache.flink.util.InstantiationUtil;
+import static org.apache.flink.util.Preconditions.checkNotNull;
public class CopyableValueSerializer<T extends CopyableValue<T>> extends TypeSerializer<T> {
@@ -39,7 +39,7 @@ public class CopyableValueSerializer<T extends CopyableValue<T>> extends TypeSer
public CopyableValueSerializer(Class<T> valueClass) {
- this.valueClass = Preconditions.checkNotNull(valueClass);
+ this.valueClass = checkNotNull(valueClass);
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializer.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializer.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializer.java
index de24956..9958540 100644
--- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializer.java
@@ -32,7 +32,6 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
-import com.google.common.base.Preconditions;
import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.common.typeutils.TypeSerializer;
@@ -40,6 +39,7 @@ import org.apache.flink.api.java.typeutils.TypeExtractor;
import org.apache.flink.core.memory.DataInputView;
import org.apache.flink.core.memory.DataOutputView;
+import static org.apache.flink.util.Preconditions.checkNotNull;
public final class PojoSerializer<T> extends TypeSerializer<T> {
@@ -75,11 +75,11 @@ public final class PojoSerializer<T> extends TypeSerializer<T> {
Field[] fields,
ExecutionConfig executionConfig) {
- this.clazz = Preconditions.checkNotNull(clazz);
- this.fieldSerializers = (TypeSerializer<Object>[]) Preconditions.checkNotNull(fieldSerializers);
- this.fields = Preconditions.checkNotNull(fields);
+ this.clazz = checkNotNull(clazz);
+ this.fieldSerializers = (TypeSerializer<Object>[]) checkNotNull(fieldSerializers);
+ this.fields = checkNotNull(fields);
this.numFields = fieldSerializers.length;
- this.executionConfig = Preconditions.checkNotNull(executionConfig);
+ this.executionConfig = checkNotNull(executionConfig);
LinkedHashSet<Class<?>> registeredPojoTypes = executionConfig.getRegisteredPojoTypes();
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/TupleSerializerBase.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/TupleSerializerBase.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/TupleSerializerBase.java
index 5b5d462..5a93cc5 100644
--- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/TupleSerializerBase.java
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/TupleSerializerBase.java
@@ -18,7 +18,6 @@
package org.apache.flink.api.java.typeutils.runtime;
-import com.google.common.base.Preconditions;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.core.memory.DataInputView;
import org.apache.flink.core.memory.DataOutputView;
@@ -27,6 +26,7 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.Objects;
+import static org.apache.flink.util.Preconditions.checkNotNull;
public abstract class TupleSerializerBase<T> extends TypeSerializer<T> {
@@ -42,8 +42,8 @@ public abstract class TupleSerializerBase<T> extends TypeSerializer<T> {
@SuppressWarnings("unchecked")
public TupleSerializerBase(Class<T> tupleClass, TypeSerializer<?>[] fieldSerializers) {
- this.tupleClass = Preconditions.checkNotNull(tupleClass);
- this.fieldSerializers = (TypeSerializer<Object>[]) Preconditions.checkNotNull(fieldSerializers);
+ this.tupleClass = checkNotNull(tupleClass);
+ this.fieldSerializers = (TypeSerializer<Object>[]) checkNotNull(fieldSerializers);
this.arity = fieldSerializers.length;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/ValueSerializer.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/ValueSerializer.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/ValueSerializer.java
index 9329866..73dc0fc 100644
--- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/ValueSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/ValueSerializer.java
@@ -20,7 +20,6 @@ package org.apache.flink.api.java.typeutils.runtime;
import java.io.IOException;
-import com.google.common.base.Preconditions;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.core.memory.DataInputView;
import org.apache.flink.core.memory.DataOutputView;
@@ -30,6 +29,8 @@ import org.apache.flink.util.InstantiationUtil;
import com.esotericsoftware.kryo.Kryo;
import org.objenesis.strategy.StdInstantiatorStrategy;
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
/**
* Serializer for {@link Value} types. Uses the value's serialization methods, and uses
* Kryo for deep object copies.
@@ -49,7 +50,7 @@ public class ValueSerializer<T extends Value> extends TypeSerializer<T> {
// --------------------------------------------------------------------------------------------
public ValueSerializer(Class<T> type) {
- this.type = Preconditions.checkNotNull(type);
+ this.type = checkNotNull(type);
}
// --------------------------------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/760a0d9e/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializer.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializer.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializer.java
index 276ffc4..d5c2f67 100644
--- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializer.java
@@ -25,9 +25,9 @@ import com.esotericsoftware.kryo.factories.ReflectionSerializerFactory;
import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output;
import com.esotericsoftware.kryo.serializers.JavaSerializer;
-import com.google.common.base.Preconditions;
import org.apache.avro.generic.GenericData;
+
import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.java.typeutils.runtime.DataInputViewStream;
@@ -36,7 +36,9 @@ import org.apache.flink.api.java.typeutils.runtime.NoFetchingInput;
import org.apache.flink.api.java.typeutils.runtime.kryo.Serializers.SpecificInstanceCollectionSerializerForArrayList;
import org.apache.flink.core.memory.DataInputView;
import org.apache.flink.core.memory.DataOutputView;
+
import org.objenesis.strategy.StdInstantiatorStrategy;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -52,6 +54,8 @@ import java.util.LinkedHashSet;
import java.util.Map;
import java.util.Objects;
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
/**
* A type serializer that serializes its type using the Kryo serialization
* framework (https://github.com/EsotericSoftware/kryo).
@@ -92,7 +96,7 @@ public class KryoSerializer<T> extends TypeSerializer<T> {
// ------------------------------------------------------------------------
public KryoSerializer(Class<T> type, ExecutionConfig executionConfig){
- this.type = Preconditions.checkNotNull(type);
+ this.type = checkNotNull(type);
this.defaultSerializers = executionConfig.getDefaultKryoSerializers();
this.defaultSerializerClasses = executionConfig.getDefaultKryoSerializerClasses();
[12/12] flink git commit: [FLINK-3759] [table] Throw exception if
NULL literal is encountered in non-null mode.
Posted by fh...@apache.org.
[FLINK-3759] [table] Throw exception if NULL literal is encountered in non-null mode.
This closes #1892
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/722155a1
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/722155a1
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/722155a1
Branch: refs/heads/master
Commit: 722155a1fb95ddb45b6ee4c6cc6d0438cdbafac6
Parents: 1acd844
Author: twalthr <tw...@apache.org>
Authored: Fri Apr 15 11:05:36 2016 +0200
Committer: Fabian Hueske <fh...@apache.org>
Committed: Fri Apr 15 19:38:15 2016 +0200
----------------------------------------------------------------------
.../flink/api/table/codegen/CodeGenerator.scala | 11 +++----
.../api/java/table/test/ExpressionsITCase.java | 24 +++++++++------
.../api/scala/sql/test/ExpressionsITCase.scala | 32 ++++++++++++--------
.../scala/table/test/ExpressionsITCase.scala | 22 ++++++++++----
4 files changed, 55 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/722155a1/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/codegen/CodeGenerator.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/codegen/CodeGenerator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/codegen/CodeGenerator.scala
index c336c82..7b3b02a 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/codegen/CodeGenerator.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/codegen/CodeGenerator.scala
@@ -907,18 +907,15 @@ class CodeGenerator(
val resultTypeTerm = primitiveTypeTermForTypeInfo(resultType)
val defaultValue = primitiveDefaultValue(resultType)
- val wrappedCode = if (nullCheck) {
- s"""
+ if (nullCheck) {
+ val wrappedCode = s"""
|$resultTypeTerm $resultTerm = $defaultValue;
|boolean $nullTerm = true;
|""".stripMargin
+ GeneratedExpression(resultTerm, nullTerm, wrappedCode, resultType)
} else {
- s"""
- |$resultTypeTerm $resultTerm = $defaultValue;
- |""".stripMargin
+ throw new CodeGenException("Null literals are not allowed if nullCheck is disabled.")
}
-
- GeneratedExpression(resultTerm, nullTerm, wrappedCode, resultType)
}
// ----------------------------------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/722155a1/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/ExpressionsITCase.java
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/ExpressionsITCase.java b/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/ExpressionsITCase.java
index 7980fe3..879c74f 100644
--- a/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/ExpressionsITCase.java
+++ b/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/ExpressionsITCase.java
@@ -27,7 +27,9 @@ import org.apache.flink.api.java.operators.DataSource;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.api.table.TableEnvironment;
+import org.apache.flink.api.table.codegen.CodeGenException;
import org.apache.flink.api.table.test.utils.TableProgramsTestBase;
+import static org.junit.Assert.fail;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
@@ -104,7 +106,7 @@ public class ExpressionsITCase extends TableProgramsTestBase {
@Test
public void testNullLiteral() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = getJavaTableEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env, config());
DataSource<Tuple2<Integer, Integer>> input =
env.fromElements(new Tuple2<>(1, 0));
@@ -114,16 +116,20 @@ public class ExpressionsITCase extends TableProgramsTestBase {
Table result = table.select("a, b, Null(INT), Null(STRING) === ''");
- DataSet<Row> ds = tableEnv.toDataSet(result, Row.class);
- List<Row> results = ds.collect();
- String expected;
- if (getConfig().getNullCheck()) {
- expected = "1,0,null,null";
+ try {
+ DataSet<Row> ds = tableEnv.toDataSet(result, Row.class);
+ if (!config().getNullCheck()) {
+ fail("Exception expected if null check is disabled.");
+ }
+ List<Row> results = ds.collect();
+ String expected = "1,0,null,null";
+ compareResultAsText(results, expected);
}
- else {
- expected = "1,0,-1,true";
+ catch (CodeGenException e) {
+ if (config().getNullCheck()) {
+ throw e;
+ }
}
- compareResultAsText(results, expected);
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/722155a1/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/sql/test/ExpressionsITCase.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/sql/test/ExpressionsITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/sql/test/ExpressionsITCase.scala
index 1d72c5d..fb69d08 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/sql/test/ExpressionsITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/sql/test/ExpressionsITCase.scala
@@ -19,13 +19,14 @@
package org.apache.flink.api.scala.sql.test
import org.apache.flink.api.scala.{ExecutionEnvironment, _}
+import org.apache.flink.api.table.{TableEnvironment, Row}
import org.apache.flink.api.scala.table._
-import org.apache.flink.api.table.Row
-import org.apache.flink.api.table.plan.TranslationContext
+import org.apache.flink.api.table.codegen.CodeGenException
import org.apache.flink.api.table.test.utils.TableProgramsTestBase
import org.apache.flink.api.table.test.utils.TableProgramsTestBase.TableConfigMode
import org.apache.flink.test.util.MultipleProgramsTestBase.TestExecutionMode
import org.apache.flink.test.util.TestBaseUtils
+import org.junit.Assert._
import org.junit.Test
import org.junit.runner.RunWith
import org.junit.runners.Parameterized
@@ -42,23 +43,30 @@ class ExpressionsITCase(
def testNullLiteral(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
- val tEnv = getScalaTableEnvironment
- TranslationContext.reset()
+ val tEnv = TableEnvironment.getTableEnvironment(env, config)
val sqlQuery = "SELECT a, b, CAST(NULL AS INT), CAST(NULL AS VARCHAR) = '' FROM MyTable"
- val ds = env.fromElements((1, 0))
- tEnv.registerDataSet("MyTable", ds, 'a, 'b)
+ val t = env.fromElements((1, 0))
+ tEnv.registerDataSet("MyTable", t, 'a, 'b)
val result = tEnv.sql(sqlQuery)
- val expected = if (getConfig.getNullCheck) {
- "1,0,null,null"
- } else {
- "1,0,-1,true"
+ try {
+ val ds = result.toDataSet[Row]
+ if (!config.getNullCheck) {
+ fail("Exception expected if null check is disabled.")
+ }
+ val results = ds.collect()
+ val expected = "1,0,null,null"
+ TestBaseUtils.compareResultAsText(results.asJava, expected)
+ }
+ catch {
+ case e: CodeGenException =>
+ if (config.getNullCheck) {
+ throw e
+ }
}
- val results = result.toDataSet[Row](getConfig).collect()
- TestBaseUtils.compareResultAsText(results.asJava, expected)
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/722155a1/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/ExpressionsITCase.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/ExpressionsITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/ExpressionsITCase.scala
index 9e908dc..0e07eaa 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/ExpressionsITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/table/test/ExpressionsITCase.scala
@@ -23,6 +23,7 @@ import java.util.Date
import org.apache.flink.api.common.typeinfo.BasicTypeInfo
import org.apache.flink.api.scala._
import org.apache.flink.api.scala.table._
+import org.apache.flink.api.table.codegen.CodeGenException
import org.apache.flink.api.table.expressions.Null
import org.apache.flink.api.table.{TableEnvironment, Row}
import org.apache.flink.api.table.expressions.Literal
@@ -30,6 +31,7 @@ import org.apache.flink.api.table.test.utils.TableProgramsTestBase
import org.apache.flink.api.table.test.utils.TableProgramsTestBase.TableConfigMode
import org.apache.flink.test.util.MultipleProgramsTestBase.TestExecutionMode
import org.apache.flink.test.util.TestBaseUtils
+import org.junit.Assert._
import org.junit._
import org.junit.runner.RunWith
import org.junit.runners.Parameterized
@@ -110,13 +112,21 @@ class ExpressionsITCase(
Null(BasicTypeInfo.INT_TYPE_INFO),
Null(BasicTypeInfo.STRING_TYPE_INFO) === "")
- val expected = if (config.getNullCheck) {
- "1,0,null,null"
- } else {
- "1,0,-1,true"
+ try {
+ val ds = t.toDataSet[Row]
+ if (!config.getNullCheck) {
+ fail("Exception expected if null check is disabled.")
+ }
+ val results = ds.collect()
+ val expected = "1,0,null,null"
+ TestBaseUtils.compareResultAsText(results.asJava, expected)
+ }
+ catch {
+ case e: CodeGenException =>
+ if (config.getNullCheck) {
+ throw e
+ }
}
- val results = t.toDataSet[Row].collect()
- TestBaseUtils.compareResultAsText(results.asJava, expected)
}
// Date literals not yet supported
[08/12] flink git commit: [FLINK-3738] [table] Refactor
TableEnvironments. Remove Translators and TranslationContext.
Posted by fh...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/AsITCase.java
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/AsITCase.java b/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/AsITCase.java
deleted file mode 100644
index faa6b0c..0000000
--- a/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/AsITCase.java
+++ /dev/null
@@ -1,416 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.api.java.table.test;
-
-import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.java.tuple.Tuple3;
-import org.apache.flink.api.java.tuple.Tuple4;
-import org.apache.flink.api.java.typeutils.TupleTypeInfo;
-import org.apache.flink.api.table.Table;
-import org.apache.flink.api.table.Row;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.table.TableEnvironment;
-import org.apache.flink.api.table.test.utils.TableProgramsTestBase;
-import org.apache.flink.test.javaApiOperators.util.CollectionDataSets;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-import java.util.ArrayList;
-import java.util.List;
-
-@RunWith(Parameterized.class)
-public class AsITCase extends TableProgramsTestBase {
-
- public AsITCase(TestExecutionMode mode, TableConfigMode configMode){
- super(mode, configMode);
- }
-
- @Test
- public void testAsFromTuple() throws Exception {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = getJavaTableEnvironment();
-
- Table table = tableEnv
- .fromDataSet(CollectionDataSets.get3TupleDataSet(env), "a, b, c")
- .select("a, b, c");
-
- DataSet<Row> ds = tableEnv.toDataSet(table, Row.class);
- List<Row> results = ds.collect();
- String expected = "1,1,Hi\n" + "2,2,Hello\n" + "3,2,Hello world\n" +
- "4,3,Hello world, how are you?\n" + "5,3,I am fine.\n" + "6,3,Luke Skywalker\n" +
- "7,4,Comment#1\n" + "8,4,Comment#2\n" + "9,4,Comment#3\n" + "10,4,Comment#4\n" +
- "11,5,Comment#5\n" + "12,5,Comment#6\n" + "13,5,Comment#7\n" +
- "14,5,Comment#8\n" + "15,5,Comment#9\n" + "16,6,Comment#10\n" +
- "17,6,Comment#11\n" + "18,6,Comment#12\n" + "19,6,Comment#13\n" +
- "20,6,Comment#14\n" + "21,6,Comment#15\n";
- compareResultAsText(results, expected);
- }
-
- @Test
- public void testAsFromAndToTuple() throws Exception {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = getJavaTableEnvironment();
-
- Table table = tableEnv
- .fromDataSet(CollectionDataSets.get3TupleDataSet(env), "a, b, c")
- .select("a, b, c");
-
- TypeInformation<?> ti = new TupleTypeInfo<Tuple3<Integer, Long, String>>(
- BasicTypeInfo.INT_TYPE_INFO,
- BasicTypeInfo.LONG_TYPE_INFO,
- BasicTypeInfo.STRING_TYPE_INFO);
-
- DataSet<?> ds = tableEnv.toDataSet(table, ti);
- List<?> results = ds.collect();
- String expected = "(1,1,Hi)\n" + "(2,2,Hello)\n" + "(3,2,Hello world)\n" +
- "(4,3,Hello world, how are you?)\n" + "(5,3,I am fine.)\n" + "(6,3,Luke Skywalker)\n" +
- "(7,4,Comment#1)\n" + "(8,4,Comment#2)\n" + "(9,4,Comment#3)\n" + "(10,4,Comment#4)\n" +
- "(11,5,Comment#5)\n" + "(12,5,Comment#6)\n" + "(13,5,Comment#7)\n" +
- "(14,5,Comment#8)\n" + "(15,5,Comment#9)\n" + "(16,6,Comment#10)\n" +
- "(17,6,Comment#11)\n" + "(18,6,Comment#12)\n" + "(19,6,Comment#13)\n" +
- "(20,6,Comment#14)\n" + "(21,6,Comment#15)\n";
- compareResultAsText(results, expected);
- }
-
- @Test
- public void testAsFromTupleToPojo() throws Exception {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = getJavaTableEnvironment();
-
- List<Tuple4<String, Integer, Double, String>> data = new ArrayList<>();
- data.add(new Tuple4<>("Rofl", 1, 1.0, "Hi"));
- data.add(new Tuple4<>("lol", 2, 1.0, "Hi"));
- data.add(new Tuple4<>("Test me", 4, 3.33, "Hello world"));
-
- Table table = tableEnv
- .fromDataSet(env.fromCollection(data), "a, b, c, d")
- .select("a, b, c, d");
-
- DataSet<SmallPojo2> ds = tableEnv.toDataSet(table, SmallPojo2.class);
- List<SmallPojo2> results = ds.collect();
- String expected = "Rofl,1,1.0,Hi\n" + "lol,2,1.0,Hi\n" + "Test me,4,3.33,Hello world\n";
- compareResultAsText(results, expected);
- }
-
- @Test
- public void testAsFromPojo() throws Exception {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = getJavaTableEnvironment();
-
- List<SmallPojo> data = new ArrayList<>();
- data.add(new SmallPojo("Peter", 28, 4000.00, "Sales"));
- data.add(new SmallPojo("Anna", 56, 10000.00, "Engineering"));
- data.add(new SmallPojo("Lucy", 42, 6000.00, "HR"));
-
- Table table = tableEnv
- .fromDataSet(env.fromCollection(data),
- "department AS a, " +
- "age AS b, " +
- "salary AS c, " +
- "name AS d")
- .select("a, b, c, d");
-
- DataSet<Row> ds = tableEnv.toDataSet(table, Row.class);
- List<Row> results = ds.collect();
- String expected =
- "Sales,28,4000.0,Peter\n" +
- "Engineering,56,10000.0,Anna\n" +
- "HR,42,6000.0,Lucy\n";
- compareResultAsText(results, expected);
- }
-
- @Test
- public void testAsFromPrivateFieldsPojo() throws Exception {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = getJavaTableEnvironment();
-
- List<PrivateSmallPojo> data = new ArrayList<>();
- data.add(new PrivateSmallPojo("Peter", 28, 4000.00, "Sales"));
- data.add(new PrivateSmallPojo("Anna", 56, 10000.00, "Engineering"));
- data.add(new PrivateSmallPojo("Lucy", 42, 6000.00, "HR"));
-
- Table table = tableEnv
- .fromDataSet(env.fromCollection(data),
- "department AS a, " +
- "age AS b, " +
- "salary AS c, " +
- "name AS d")
- .select("a, b, c, d");
-
- DataSet<Row> ds = tableEnv.toDataSet(table, Row.class);
- List<Row> results = ds.collect();
- String expected =
- "Sales,28,4000.0,Peter\n" +
- "Engineering,56,10000.0,Anna\n" +
- "HR,42,6000.0,Lucy\n";
- compareResultAsText(results, expected);
- }
-
- @Test
- public void testAsFromAndToPojo() throws Exception {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = getJavaTableEnvironment();
-
- List<SmallPojo> data = new ArrayList<>();
- data.add(new SmallPojo("Peter", 28, 4000.00, "Sales"));
- data.add(new SmallPojo("Anna", 56, 10000.00, "Engineering"));
- data.add(new SmallPojo("Lucy", 42, 6000.00, "HR"));
-
- Table table = tableEnv
- .fromDataSet(env.fromCollection(data),
- "department AS a, " +
- "age AS b, " +
- "salary AS c, " +
- "name AS d")
- .select("a, b, c, d");
-
- DataSet<SmallPojo2> ds = tableEnv.toDataSet(table, SmallPojo2.class);
- List<SmallPojo2> results = ds.collect();
- String expected =
- "Sales,28,4000.0,Peter\n" +
- "Engineering,56,10000.0,Anna\n" +
- "HR,42,6000.0,Lucy\n";
- compareResultAsText(results, expected);
- }
-
- @Test
- public void testAsFromAndToPrivateFieldPojo() throws Exception {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = getJavaTableEnvironment();
-
- List<PrivateSmallPojo> data = new ArrayList<>();
- data.add(new PrivateSmallPojo("Peter", 28, 4000.00, "Sales"));
- data.add(new PrivateSmallPojo("Anna", 56, 10000.00, "Engineering"));
- data.add(new PrivateSmallPojo("Lucy", 42, 6000.00, "HR"));
-
- Table table = tableEnv
- .fromDataSet(env.fromCollection(data),
- "department AS a, " +
- "age AS b, " +
- "salary AS c, " +
- "name AS d")
- .select("a, b, c, d");
-
- DataSet<PrivateSmallPojo2> ds = tableEnv.toDataSet(table, PrivateSmallPojo2.class);
- List<PrivateSmallPojo2> results = ds.collect();
- String expected =
- "Sales,28,4000.0,Peter\n" +
- "Engineering,56,10000.0,Anna\n" +
- "HR,42,6000.0,Lucy\n";
- compareResultAsText(results, expected);
- }
-
- @Test(expected = IllegalArgumentException.class)
- public void testAsWithToFewFields() throws Exception {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = getJavaTableEnvironment();
-
- // Must fail. Not enough field names specified.
- tableEnv.fromDataSet(CollectionDataSets.get3TupleDataSet(env), "a, b");
- }
-
- @Test(expected = IllegalArgumentException.class)
- public void testAsWithToManyFields() throws Exception {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = getJavaTableEnvironment();
-
- // Must fail. Too many field names specified.
- tableEnv.fromDataSet(CollectionDataSets.get3TupleDataSet(env), "a, b, c, d");
- }
-
- @Test(expected = IllegalArgumentException.class)
- public void testAsWithAmbiguousFields() throws Exception {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = getJavaTableEnvironment();
-
- // Must fail. Specified field names are not unique.
- tableEnv.fromDataSet(CollectionDataSets.get3TupleDataSet(env), "a, b, b");
- }
-
- @Test(expected = IllegalArgumentException.class)
- public void testAsWithNonFieldReference1() throws Exception {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = getJavaTableEnvironment();
-
- // Must fail. as() does only allow field name expressions
- tableEnv.fromDataSet(CollectionDataSets.get3TupleDataSet(env), "a + 1, b, c");
- }
-
- @Test(expected = IllegalArgumentException.class)
- public void testAsWithNonFieldReference2() throws Exception {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = getJavaTableEnvironment();
-
- // Must fail. as() does only allow field name expressions
- tableEnv.fromDataSet(CollectionDataSets.get3TupleDataSet(env), "a as foo, b, c");
- }
-
- // --------------------------------------------------------------------------------------------
-
- @SuppressWarnings("unused")
- public static class SmallPojo {
-
- public SmallPojo() { }
-
- public SmallPojo(String name, int age, double salary, String department) {
- this.name = name;
- this.age = age;
- this.salary = salary;
- this.department = department;
- }
-
- public String name;
- public int age;
- public double salary;
- public String department;
- }
-
- @SuppressWarnings("unused")
- public static class PrivateSmallPojo {
-
- public PrivateSmallPojo() { }
-
- public PrivateSmallPojo(String name, int age, double salary, String department) {
- this.name = name;
- this.age = age;
- this.salary = salary;
- this.department = department;
- }
-
- private String name;
- private int age;
- private double salary;
- private String department;
-
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- public int getAge() {
- return age;
- }
-
- public void setAge(int age) {
- this.age = age;
- }
-
- public double getSalary() {
- return salary;
- }
-
- public void setSalary(double salary) {
- this.salary = salary;
- }
-
- public String getDepartment() {
- return department;
- }
-
- public void setDepartment(String department) {
- this.department = department;
- }
- }
-
- @SuppressWarnings("unused")
- public static class SmallPojo2 {
-
- public SmallPojo2() { }
-
- public SmallPojo2(String a, int b, double c, String d) {
- this.a = a;
- this.b = b;
- this.c = c;
- this.d = d;
- }
-
- public String a;
- public int b;
- public double c;
- public String d;
-
- @Override
- public String toString() {
- return a + "," + b + "," + c + "," + d;
- }
- }
-
- @SuppressWarnings("unused")
- public static class PrivateSmallPojo2 {
-
- public PrivateSmallPojo2() { }
-
- public PrivateSmallPojo2(String a, int b, double c, String d) {
- this.a = a;
- this.b = b;
- this.c = c;
- this.d = d;
- }
-
- private String a;
- private int b;
- private double c;
- private String d;
-
- public String getA() {
- return a;
- }
-
- public void setA(String a) {
- this.a = a;
- }
-
- public int getB() {
- return b;
- }
-
- public void setB(int b) {
- this.b = b;
- }
-
- public double getC() {
- return c;
- }
-
- public void setC(double c) {
- this.c = c;
- }
-
- public String getD() {
- return d;
- }
-
- public void setD(String d) {
- this.d = d;
- }
-
- @Override
- public String toString() {
- return a + "," + b + "," + c + "," + d;
- }
- }
-
-}
-
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/CastingITCase.java
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/CastingITCase.java b/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/CastingITCase.java
index 5a4adbc..2a17c12 100644
--- a/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/CastingITCase.java
+++ b/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/CastingITCase.java
@@ -22,7 +22,7 @@ import java.util.List;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.flink.api.java.operators.DataSource;
-import org.apache.flink.api.java.table.TableEnvironment;
+import org.apache.flink.api.java.table.BatchTableEnvironment;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.api.java.tuple.Tuple4;
@@ -30,7 +30,7 @@ import org.apache.flink.api.java.tuple.Tuple6;
import org.apache.flink.api.java.tuple.Tuple8;
import org.apache.flink.api.table.Row;
import org.apache.flink.api.table.Table;
-import org.apache.flink.api.table.codegen.CodeGenException;
+import org.apache.flink.api.table.TableEnvironment;
import org.apache.flink.api.table.test.utils.TableProgramsTestBase;
import org.junit.Ignore;
import org.junit.Test;
@@ -47,7 +47,7 @@ public class CastingITCase extends TableProgramsTestBase {
@Test
public void testNumericAutocastInArithmetic() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = getJavaTableEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env, config());
DataSource<Tuple8<Byte, Short, Integer, Long, Float, Double, Long, Double>> input =
env.fromElements(new Tuple8<>((byte) 1, (short) 1, 1, 1L, 1.0f, 1.0d, 1L, 1001.1));
@@ -67,7 +67,7 @@ public class CastingITCase extends TableProgramsTestBase {
@Test
public void testNumericAutocastInComparison() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = getJavaTableEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env, config());
DataSource<Tuple6<Byte, Short, Integer, Long, Float, Double>> input =
env.fromElements(
@@ -89,7 +89,7 @@ public class CastingITCase extends TableProgramsTestBase {
@Test
public void testCasting() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = new TableEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env, config());
DataSource<Tuple4<Integer, Double, Long, Boolean>> input =
env.fromElements(new Tuple4<>(1, 0.0, 1L, true));
@@ -122,7 +122,7 @@ public class CastingITCase extends TableProgramsTestBase {
@Test
public void testCastFromString() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = new TableEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env, config());
DataSource<Tuple3<String, String, String>> input =
env.fromElements(new Tuple3<>("1", "true", "2.0"));
@@ -143,7 +143,7 @@ public class CastingITCase extends TableProgramsTestBase {
@Test
public void testCastDateFromString() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = new TableEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env, config());
DataSource<Tuple4<String, String, String, String>> input =
env.fromElements(new Tuple4<>("2011-05-03", "15:51:36", "2011-05-03 15:51:36.000", "1446473775"));
@@ -166,7 +166,7 @@ public class CastingITCase extends TableProgramsTestBase {
@Test
public void testCastDateToStringAndLong() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = new TableEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env, config());
DataSource<Tuple2<String, String>> input =
env.fromElements(new Tuple2<>("2011-05-03 15:51:36.000", "1304437896000"));
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/DistinctITCase.java
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/DistinctITCase.java b/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/DistinctITCase.java
index b7d689d..aca0c30 100644
--- a/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/DistinctITCase.java
+++ b/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/DistinctITCase.java
@@ -21,11 +21,12 @@ package org.apache.flink.api.java.table.test;
import java.util.List;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.table.TableEnvironment;
+import org.apache.flink.api.java.table.BatchTableEnvironment;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.api.java.tuple.Tuple5;
import org.apache.flink.api.table.Row;
import org.apache.flink.api.table.Table;
+import org.apache.flink.api.table.TableEnvironment;
import org.apache.flink.test.javaApiOperators.util.CollectionDataSets;
import org.apache.flink.test.util.MultipleProgramsTestBase;
import org.junit.Test;
@@ -42,7 +43,7 @@ public class DistinctITCase extends MultipleProgramsTestBase {
@Test
public void testDistinct() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = new TableEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
DataSet<Tuple3<Integer, Long, String>> input = CollectionDataSets.get3TupleDataSet(env);
@@ -59,7 +60,7 @@ public class DistinctITCase extends MultipleProgramsTestBase {
@Test
public void testDistinctAfterAggregate() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = new TableEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
DataSet<Tuple5<Integer, Long, Integer, String, Long>> input = CollectionDataSets.get5TupleDataSet(env);
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/ExpressionsITCase.java
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/ExpressionsITCase.java b/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/ExpressionsITCase.java
index 996542d..7980fe3 100644
--- a/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/ExpressionsITCase.java
+++ b/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/ExpressionsITCase.java
@@ -22,10 +22,11 @@ import org.apache.flink.api.table.Table;
import org.apache.flink.api.table.Row;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.table.TableEnvironment;
+import org.apache.flink.api.java.table.BatchTableEnvironment;
import org.apache.flink.api.java.operators.DataSource;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.tuple.Tuple3;
+import org.apache.flink.api.table.TableEnvironment;
import org.apache.flink.api.table.test.utils.TableProgramsTestBase;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -43,7 +44,7 @@ public class ExpressionsITCase extends TableProgramsTestBase {
@Test
public void testArithmetic() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = getJavaTableEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env, config());
DataSource<Tuple2<Integer, Integer>> input =
env.fromElements(new Tuple2<>(5, 10));
@@ -63,7 +64,7 @@ public class ExpressionsITCase extends TableProgramsTestBase {
@Test
public void testLogic() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = getJavaTableEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env, config());
DataSource<Tuple2<Integer, Boolean>> input =
env.fromElements(new Tuple2<>(5, true));
@@ -83,7 +84,7 @@ public class ExpressionsITCase extends TableProgramsTestBase {
@Test
public void testComparisons() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = getJavaTableEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env, config());
DataSource<Tuple3<Integer, Integer, Integer>> input =
env.fromElements(new Tuple3<>(5, 5, 4));
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/FilterITCase.java
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/FilterITCase.java b/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/FilterITCase.java
index 45bd7e4..30b5aab 100644
--- a/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/FilterITCase.java
+++ b/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/FilterITCase.java
@@ -22,9 +22,9 @@ import org.apache.flink.api.table.Table;
import org.apache.flink.api.table.Row;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.table.TableEnvironment;
+import org.apache.flink.api.java.table.BatchTableEnvironment;
import org.apache.flink.api.java.tuple.Tuple3;
-import org.apache.flink.api.table.codegen.CodeGenException;
+import org.apache.flink.api.table.TableEnvironment;
import org.apache.flink.api.table.test.utils.TableProgramsTestBase;
import org.apache.flink.test.javaApiOperators.util.CollectionDataSets;
import org.junit.Test;
@@ -43,7 +43,7 @@ public class FilterITCase extends TableProgramsTestBase {
@Test
public void testAllRejectingFilter() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = getJavaTableEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env, config());
DataSet<Tuple3<Integer, Long, String>> input = CollectionDataSets.get3TupleDataSet(env);
Table table = tableEnv.fromDataSet(input, "a, b, c");
@@ -60,7 +60,7 @@ public class FilterITCase extends TableProgramsTestBase {
@Test
public void testAllPassingFilter() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = getJavaTableEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env, config());
DataSet<Tuple3<Integer, Long, String>> input = CollectionDataSets.get3TupleDataSet(env);
Table table = tableEnv.fromDataSet(input, "a, b, c");
@@ -83,7 +83,7 @@ public class FilterITCase extends TableProgramsTestBase {
@Test
public void testFilterOnIntegerTupleField() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = getJavaTableEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env, config());
DataSet<Tuple3<Integer, Long, String>> input = CollectionDataSets.get3TupleDataSet(env);
Table table = tableEnv.fromDataSet(input, "a, b, c");
@@ -102,7 +102,7 @@ public class FilterITCase extends TableProgramsTestBase {
@Test
public void testNotEquals() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = getJavaTableEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env, config());
DataSet<Tuple3<Integer, Long, String>> input = CollectionDataSets.get3TupleDataSet(env);
Table table = tableEnv.fromDataSet(input, "a, b, c");
@@ -121,7 +121,7 @@ public class FilterITCase extends TableProgramsTestBase {
@Test
public void testDisjunctivePreds() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = getJavaTableEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env, config());
DataSet<Tuple3<Integer, Long, String>> input = CollectionDataSets.get3TupleDataSet(env);
Table table = tableEnv.fromDataSet(input, "a, b, c");
@@ -138,7 +138,7 @@ public class FilterITCase extends TableProgramsTestBase {
@Test
public void testIntegerBiggerThan128() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = getJavaTableEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env, config());
DataSet<Tuple3<Integer, Long, String>> input = env.fromElements(new Tuple3<>(300, 1L, "Hello"));
Table table = tableEnv.fromDataSet(input, "a, b, c");
@@ -155,7 +155,7 @@ public class FilterITCase extends TableProgramsTestBase {
@Test(expected = IllegalArgumentException.class)
public void testFilterInvalidField() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = getJavaTableEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env, config());
DataSet<Tuple3<Integer, Long, String>> input = CollectionDataSets.get3TupleDataSet(env);
Table table = tableEnv.fromDataSet(input, "a, b, c");
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/FromDataSetITCase.java
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/FromDataSetITCase.java b/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/FromDataSetITCase.java
new file mode 100644
index 0000000..64a4092
--- /dev/null
+++ b/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/FromDataSetITCase.java
@@ -0,0 +1,417 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.java.table.test;
+
+import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.java.tuple.Tuple3;
+import org.apache.flink.api.java.tuple.Tuple4;
+import org.apache.flink.api.java.typeutils.TupleTypeInfo;
+import org.apache.flink.api.table.Table;
+import org.apache.flink.api.table.Row;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.table.BatchTableEnvironment;
+import org.apache.flink.api.table.TableEnvironment;
+import org.apache.flink.api.table.test.utils.TableProgramsTestBase;
+import org.apache.flink.test.javaApiOperators.util.CollectionDataSets;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.util.ArrayList;
+import java.util.List;
+
+@RunWith(Parameterized.class)
+public class FromDataSetITCase extends TableProgramsTestBase {
+
+ public FromDataSetITCase(TestExecutionMode mode, TableConfigMode configMode){
+ super(mode, configMode);
+ }
+
+ @Test
+ public void testAsFromTuple() throws Exception {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env, config());
+
+ Table table = tableEnv
+ .fromDataSet(CollectionDataSets.get3TupleDataSet(env), "a, b, c")
+ .select("a, b, c");
+
+ DataSet<Row> ds = tableEnv.toDataSet(table, Row.class);
+ List<Row> results = ds.collect();
+ String expected = "1,1,Hi\n" + "2,2,Hello\n" + "3,2,Hello world\n" +
+ "4,3,Hello world, how are you?\n" + "5,3,I am fine.\n" + "6,3,Luke Skywalker\n" +
+ "7,4,Comment#1\n" + "8,4,Comment#2\n" + "9,4,Comment#3\n" + "10,4,Comment#4\n" +
+ "11,5,Comment#5\n" + "12,5,Comment#6\n" + "13,5,Comment#7\n" +
+ "14,5,Comment#8\n" + "15,5,Comment#9\n" + "16,6,Comment#10\n" +
+ "17,6,Comment#11\n" + "18,6,Comment#12\n" + "19,6,Comment#13\n" +
+ "20,6,Comment#14\n" + "21,6,Comment#15\n";
+ compareResultAsText(results, expected);
+ }
+
+ @Test
+ public void testAsFromAndToTuple() throws Exception {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env, config());
+
+ Table table = tableEnv
+ .fromDataSet(CollectionDataSets.get3TupleDataSet(env), "a, b, c")
+ .select("a, b, c");
+
+ TypeInformation<?> ti = new TupleTypeInfo<Tuple3<Integer, Long, String>>(
+ BasicTypeInfo.INT_TYPE_INFO,
+ BasicTypeInfo.LONG_TYPE_INFO,
+ BasicTypeInfo.STRING_TYPE_INFO);
+
+ DataSet<?> ds = tableEnv.toDataSet(table, ti);
+ List<?> results = ds.collect();
+ String expected = "(1,1,Hi)\n" + "(2,2,Hello)\n" + "(3,2,Hello world)\n" +
+ "(4,3,Hello world, how are you?)\n" + "(5,3,I am fine.)\n" + "(6,3,Luke Skywalker)\n" +
+ "(7,4,Comment#1)\n" + "(8,4,Comment#2)\n" + "(9,4,Comment#3)\n" + "(10,4,Comment#4)\n" +
+ "(11,5,Comment#5)\n" + "(12,5,Comment#6)\n" + "(13,5,Comment#7)\n" +
+ "(14,5,Comment#8)\n" + "(15,5,Comment#9)\n" + "(16,6,Comment#10)\n" +
+ "(17,6,Comment#11)\n" + "(18,6,Comment#12)\n" + "(19,6,Comment#13)\n" +
+ "(20,6,Comment#14)\n" + "(21,6,Comment#15)\n";
+ compareResultAsText(results, expected);
+ }
+
+ @Test
+ public void testAsFromTupleToPojo() throws Exception {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env, config());
+
+ List<Tuple4<String, Integer, Double, String>> data = new ArrayList<>();
+ data.add(new Tuple4<>("Rofl", 1, 1.0, "Hi"));
+ data.add(new Tuple4<>("lol", 2, 1.0, "Hi"));
+ data.add(new Tuple4<>("Test me", 4, 3.33, "Hello world"));
+
+ Table table = tableEnv
+ .fromDataSet(env.fromCollection(data), "a, b, c, d")
+ .select("a, b, c, d");
+
+ DataSet<SmallPojo2> ds = tableEnv.toDataSet(table, SmallPojo2.class);
+ List<SmallPojo2> results = ds.collect();
+ String expected = "Rofl,1,1.0,Hi\n" + "lol,2,1.0,Hi\n" + "Test me,4,3.33,Hello world\n";
+ compareResultAsText(results, expected);
+ }
+
+ @Test
+ public void testAsFromPojo() throws Exception {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env, config());
+
+ List<SmallPojo> data = new ArrayList<>();
+ data.add(new SmallPojo("Peter", 28, 4000.00, "Sales"));
+ data.add(new SmallPojo("Anna", 56, 10000.00, "Engineering"));
+ data.add(new SmallPojo("Lucy", 42, 6000.00, "HR"));
+
+ Table table = tableEnv
+ .fromDataSet(env.fromCollection(data),
+ "department AS a, " +
+ "age AS b, " +
+ "salary AS c, " +
+ "name AS d")
+ .select("a, b, c, d");
+
+ DataSet<Row> ds = tableEnv.toDataSet(table, Row.class);
+ List<Row> results = ds.collect();
+ String expected =
+ "Sales,28,4000.0,Peter\n" +
+ "Engineering,56,10000.0,Anna\n" +
+ "HR,42,6000.0,Lucy\n";
+ compareResultAsText(results, expected);
+ }
+
+ @Test
+ public void testAsFromPrivateFieldsPojo() throws Exception {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env, config());
+
+ List<PrivateSmallPojo> data = new ArrayList<>();
+ data.add(new PrivateSmallPojo("Peter", 28, 4000.00, "Sales"));
+ data.add(new PrivateSmallPojo("Anna", 56, 10000.00, "Engineering"));
+ data.add(new PrivateSmallPojo("Lucy", 42, 6000.00, "HR"));
+
+ Table table = tableEnv
+ .fromDataSet(env.fromCollection(data),
+ "department AS a, " +
+ "age AS b, " +
+ "salary AS c, " +
+ "name AS d")
+ .select("a, b, c, d");
+
+ DataSet<Row> ds = tableEnv.toDataSet(table, Row.class);
+ List<Row> results = ds.collect();
+ String expected =
+ "Sales,28,4000.0,Peter\n" +
+ "Engineering,56,10000.0,Anna\n" +
+ "HR,42,6000.0,Lucy\n";
+ compareResultAsText(results, expected);
+ }
+
+ @Test
+ public void testAsFromAndToPojo() throws Exception {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env, config());
+
+ List<SmallPojo> data = new ArrayList<>();
+ data.add(new SmallPojo("Peter", 28, 4000.00, "Sales"));
+ data.add(new SmallPojo("Anna", 56, 10000.00, "Engineering"));
+ data.add(new SmallPojo("Lucy", 42, 6000.00, "HR"));
+
+ Table table = tableEnv
+ .fromDataSet(env.fromCollection(data),
+ "department AS a, " +
+ "age AS b, " +
+ "salary AS c, " +
+ "name AS d")
+ .select("a, b, c, d");
+
+ DataSet<SmallPojo2> ds = tableEnv.toDataSet(table, SmallPojo2.class);
+ List<SmallPojo2> results = ds.collect();
+ String expected =
+ "Sales,28,4000.0,Peter\n" +
+ "Engineering,56,10000.0,Anna\n" +
+ "HR,42,6000.0,Lucy\n";
+ compareResultAsText(results, expected);
+ }
+
+ @Test
+ public void testAsFromAndToPrivateFieldPojo() throws Exception {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env, config());
+
+ List<PrivateSmallPojo> data = new ArrayList<>();
+ data.add(new PrivateSmallPojo("Peter", 28, 4000.00, "Sales"));
+ data.add(new PrivateSmallPojo("Anna", 56, 10000.00, "Engineering"));
+ data.add(new PrivateSmallPojo("Lucy", 42, 6000.00, "HR"));
+
+ Table table = tableEnv
+ .fromDataSet(env.fromCollection(data),
+ "department AS a, " +
+ "age AS b, " +
+ "salary AS c, " +
+ "name AS d")
+ .select("a, b, c, d");
+
+ DataSet<PrivateSmallPojo2> ds = tableEnv.toDataSet(table, PrivateSmallPojo2.class);
+ List<PrivateSmallPojo2> results = ds.collect();
+ String expected =
+ "Sales,28,4000.0,Peter\n" +
+ "Engineering,56,10000.0,Anna\n" +
+ "HR,42,6000.0,Lucy\n";
+ compareResultAsText(results, expected);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testAsWithToFewFields() throws Exception {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env, config());
+
+ // Must fail. Not enough field names specified.
+ tableEnv.fromDataSet(CollectionDataSets.get3TupleDataSet(env), "a, b");
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testAsWithToManyFields() throws Exception {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env, config());
+
+ // Must fail. Too many field names specified.
+ tableEnv.fromDataSet(CollectionDataSets.get3TupleDataSet(env), "a, b, c, d");
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testAsWithAmbiguousFields() throws Exception {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env, config());
+
+ // Must fail. Specified field names are not unique.
+ tableEnv.fromDataSet(CollectionDataSets.get3TupleDataSet(env), "a, b, b");
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testAsWithNonFieldReference1() throws Exception {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env, config());
+
+ // Must fail. as() does only allow field name expressions
+ tableEnv.fromDataSet(CollectionDataSets.get3TupleDataSet(env), "a + 1, b, c");
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testAsWithNonFieldReference2() throws Exception {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env, config());
+
+ // Must fail. as() does only allow field name expressions
+ tableEnv.fromDataSet(CollectionDataSets.get3TupleDataSet(env), "a as foo, b, c");
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ @SuppressWarnings("unused")
+ public static class SmallPojo {
+
+ public SmallPojo() { }
+
+ public SmallPojo(String name, int age, double salary, String department) {
+ this.name = name;
+ this.age = age;
+ this.salary = salary;
+ this.department = department;
+ }
+
+ public String name;
+ public int age;
+ public double salary;
+ public String department;
+ }
+
+ @SuppressWarnings("unused")
+ public static class PrivateSmallPojo {
+
+ public PrivateSmallPojo() { }
+
+ public PrivateSmallPojo(String name, int age, double salary, String department) {
+ this.name = name;
+ this.age = age;
+ this.salary = salary;
+ this.department = department;
+ }
+
+ private String name;
+ private int age;
+ private double salary;
+ private String department;
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public int getAge() {
+ return age;
+ }
+
+ public void setAge(int age) {
+ this.age = age;
+ }
+
+ public double getSalary() {
+ return salary;
+ }
+
+ public void setSalary(double salary) {
+ this.salary = salary;
+ }
+
+ public String getDepartment() {
+ return department;
+ }
+
+ public void setDepartment(String department) {
+ this.department = department;
+ }
+ }
+
+ @SuppressWarnings("unused")
+ public static class SmallPojo2 {
+
+ public SmallPojo2() { }
+
+ public SmallPojo2(String a, int b, double c, String d) {
+ this.a = a;
+ this.b = b;
+ this.c = c;
+ this.d = d;
+ }
+
+ public String a;
+ public int b;
+ public double c;
+ public String d;
+
+ @Override
+ public String toString() {
+ return a + "," + b + "," + c + "," + d;
+ }
+ }
+
+ @SuppressWarnings("unused")
+ public static class PrivateSmallPojo2 {
+
+ public PrivateSmallPojo2() { }
+
+ public PrivateSmallPojo2(String a, int b, double c, String d) {
+ this.a = a;
+ this.b = b;
+ this.c = c;
+ this.d = d;
+ }
+
+ private String a;
+ private int b;
+ private double c;
+ private String d;
+
+ public String getA() {
+ return a;
+ }
+
+ public void setA(String a) {
+ this.a = a;
+ }
+
+ public int getB() {
+ return b;
+ }
+
+ public void setB(int b) {
+ this.b = b;
+ }
+
+ public double getC() {
+ return c;
+ }
+
+ public void setC(double c) {
+ this.c = c;
+ }
+
+ public String getD() {
+ return d;
+ }
+
+ public void setD(String d) {
+ this.d = d;
+ }
+
+ @Override
+ public String toString() {
+ return a + "," + b + "," + c + "," + d;
+ }
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/GroupedAggregationsITCase.java
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/GroupedAggregationsITCase.java b/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/GroupedAggregationsITCase.java
index fe85678..c57c514 100644
--- a/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/GroupedAggregationsITCase.java
+++ b/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/GroupedAggregationsITCase.java
@@ -22,8 +22,9 @@ import org.apache.flink.api.table.Table;
import org.apache.flink.api.table.Row;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.table.TableEnvironment;
+import org.apache.flink.api.java.table.BatchTableEnvironment;
import org.apache.flink.api.java.tuple.Tuple3;
+import org.apache.flink.api.table.TableEnvironment;
import org.apache.flink.test.javaApiOperators.util.CollectionDataSets;
import org.apache.flink.test.util.MultipleProgramsTestBase;
import org.junit.Test;
@@ -42,7 +43,7 @@ public class GroupedAggregationsITCase extends MultipleProgramsTestBase {
@Test(expected = IllegalArgumentException.class)
public void testGroupingOnNonExistentField() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = new TableEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
DataSet<Tuple3<Integer, Long, String>> input = CollectionDataSets.get3TupleDataSet(env);
@@ -56,7 +57,7 @@ public class GroupedAggregationsITCase extends MultipleProgramsTestBase {
@Test(expected = IllegalArgumentException.class)
public void testGroupingInvalidSelection() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = new TableEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
DataSet<Tuple3<Integer, Long, String>> input = CollectionDataSets.get3TupleDataSet(env);
@@ -70,7 +71,7 @@ public class GroupedAggregationsITCase extends MultipleProgramsTestBase {
@Test
public void testGroupedAggregate() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = new TableEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
DataSet<Tuple3<Integer, Long, String>> input = CollectionDataSets.get3TupleDataSet(env);
Table table = tableEnv.fromDataSet(input, "a, b, c");
@@ -87,7 +88,7 @@ public class GroupedAggregationsITCase extends MultipleProgramsTestBase {
@Test
public void testGroupingKeyForwardIfNotUsed() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = new TableEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
DataSet<Tuple3<Integer, Long, String>> input = CollectionDataSets.get3TupleDataSet(env);
Table table = tableEnv.fromDataSet(input, "a, b, c");
@@ -105,7 +106,7 @@ public class GroupedAggregationsITCase extends MultipleProgramsTestBase {
public void testGroupNoAggregation() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = new TableEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
DataSet<Tuple3<Integer, Long, String>> input = CollectionDataSets.get3TupleDataSet(env);
Table table = tableEnv.fromDataSet(input, "a, b, c");
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/JoinITCase.java
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/JoinITCase.java b/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/JoinITCase.java
index 1622ac4..8348a4a 100644
--- a/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/JoinITCase.java
+++ b/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/JoinITCase.java
@@ -22,9 +22,10 @@ import org.apache.flink.api.table.Row;
import org.apache.flink.api.table.Table;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.table.TableEnvironment;
+import org.apache.flink.api.java.table.BatchTableEnvironment;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.api.java.tuple.Tuple5;
+import org.apache.flink.api.table.TableEnvironment;
import org.apache.flink.api.table.TableException;
import org.apache.flink.test.javaApiOperators.util.CollectionDataSets;
import org.apache.flink.test.util.MultipleProgramsTestBase;
@@ -45,7 +46,7 @@ public class JoinITCase extends MultipleProgramsTestBase {
@Test
public void testJoin() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = new TableEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
DataSet<Tuple3<Integer, Long, String>> ds1 = CollectionDataSets.getSmall3TupleDataSet(env);
DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.get5TupleDataSet(env);
@@ -64,7 +65,7 @@ public class JoinITCase extends MultipleProgramsTestBase {
@Test
public void testJoinWithFilter() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = new TableEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
DataSet<Tuple3<Integer, Long, String>> ds1 = CollectionDataSets.getSmall3TupleDataSet(env);
DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.get5TupleDataSet(env);
@@ -83,7 +84,7 @@ public class JoinITCase extends MultipleProgramsTestBase {
@Test
public void testJoinWithJoinFilter() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = new TableEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
DataSet<Tuple3<Integer, Long, String>> ds1 = CollectionDataSets.get3TupleDataSet(env);
DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.get5TupleDataSet(env);
@@ -103,7 +104,7 @@ public class JoinITCase extends MultipleProgramsTestBase {
@Test
public void testJoinWithMultipleKeys() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = new TableEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
DataSet<Tuple3<Integer, Long, String>> ds1 = CollectionDataSets.get3TupleDataSet(env);
DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.get5TupleDataSet(env);
@@ -123,7 +124,7 @@ public class JoinITCase extends MultipleProgramsTestBase {
@Test(expected = IllegalArgumentException.class)
public void testJoinNonExistingKey() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = new TableEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
DataSet<Tuple3<Integer, Long, String>> ds1 = CollectionDataSets.getSmall3TupleDataSet(env);
DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.get5TupleDataSet(env);
@@ -138,7 +139,7 @@ public class JoinITCase extends MultipleProgramsTestBase {
@Test(expected = TableException.class)
public void testJoinWithNonMatchingKeyTypes() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = new TableEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
DataSet<Tuple3<Integer, Long, String>> ds1 = CollectionDataSets.getSmall3TupleDataSet(env);
DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.get5TupleDataSet(env);
@@ -156,7 +157,7 @@ public class JoinITCase extends MultipleProgramsTestBase {
@Test(expected = IllegalArgumentException.class)
public void testJoinWithAmbiguousFields() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = new TableEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
DataSet<Tuple3<Integer, Long, String>> ds1 = CollectionDataSets.getSmall3TupleDataSet(env);
DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.get5TupleDataSet(env);
@@ -171,7 +172,7 @@ public class JoinITCase extends MultipleProgramsTestBase {
@Test
public void testJoinWithAggregation() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = new TableEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
DataSet<Tuple3<Integer, Long, String>> ds1 = CollectionDataSets.getSmall3TupleDataSet(env);
DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.get5TupleDataSet(env);
@@ -188,4 +189,20 @@ public class JoinITCase extends MultipleProgramsTestBase {
compareResultAsText(results, expected);
}
+ @Test(expected = TableException.class)
+ public void testJoinTablesFromDifferentEnvs() {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ BatchTableEnvironment tEnv1 = TableEnvironment.getTableEnvironment(env);
+ BatchTableEnvironment tEnv2 = TableEnvironment.getTableEnvironment(env);
+
+ DataSet<Tuple3<Integer, Long, String>> ds1 = CollectionDataSets.getSmall3TupleDataSet(env);
+ DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.get5TupleDataSet(env);
+
+ Table in1 = tEnv1.fromDataSet(ds1, "a, b, c");
+ Table in2 = tEnv2.fromDataSet(ds2, "d, e, f, g, h");
+
+ // Must fail. Tables are bound to different TableEnvironments.
+ in1.join(in2).where("a === d").select("g.count");
+ }
+
}
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/PojoGroupingITCase.java
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/PojoGroupingITCase.java b/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/PojoGroupingITCase.java
index b8b84f6..1109e78 100644
--- a/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/PojoGroupingITCase.java
+++ b/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/PojoGroupingITCase.java
@@ -23,9 +23,10 @@ import java.util.List;
import org.apache.flink.api.common.operators.Order;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.table.TableEnvironment;
+import org.apache.flink.api.java.table.BatchTableEnvironment;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.api.table.Table;
+import org.apache.flink.api.table.TableEnvironment;
import org.apache.flink.test.util.MultipleProgramsTestBase;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -47,7 +48,7 @@ public class PojoGroupingITCase extends MultipleProgramsTestBase {
new Tuple3<>("A", 24.0, "Y"),
new Tuple3<>("B", 1.0, "Z"));
- TableEnvironment tableEnv = new TableEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
Table table = tableEnv
.fromDataSet(data, "groupMe, value, name")
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/RegisterDataSetITCase.java
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/RegisterDataSetITCase.java b/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/RegisterDataSetITCase.java
deleted file mode 100644
index 959fb90..0000000
--- a/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/RegisterDataSetITCase.java
+++ /dev/null
@@ -1,142 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.api.java.table.test;
-
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.table.TableEnvironment;
-import org.apache.flink.api.java.tuple.Tuple3;
-import org.apache.flink.api.java.tuple.Tuple5;
-import org.apache.flink.api.table.Row;
-import org.apache.flink.api.table.Table;
-import org.apache.flink.api.table.TableException;
-import org.apache.flink.api.table.plan.TranslationContext;
-import org.apache.flink.api.table.test.utils.TableProgramsTestBase;
-import org.apache.flink.test.javaApiOperators.util.CollectionDataSets;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-import java.util.List;
-
-@RunWith(Parameterized.class)
-public class RegisterDataSetITCase extends TableProgramsTestBase {
-
- public RegisterDataSetITCase(TestExecutionMode mode, TableConfigMode configMode) {
- super(mode, configMode);
- }
-
- @Test
- public void testSimpleRegister() throws Exception {
- final String tableName = "MyTable";
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = getJavaTableEnvironment();
- TranslationContext.reset();
-
- DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
- tableEnv.registerDataSet(tableName, ds);
- Table t = tableEnv.scan(tableName);
-
- Table result = t.select("f0, f1");
-
- DataSet<Row> resultSet = tableEnv.toDataSet(result, Row.class);
- List<Row> results = resultSet.collect();
- String expected = "1,1\n" + "2,2\n" + "3,2\n" + "4,3\n" + "5,3\n" + "6,3\n" + "7,4\n" +
- "8,4\n" + "9,4\n" + "10,4\n" + "11,5\n" + "12,5\n" + "13,5\n" + "14,5\n" + "15,5\n" +
- "16,6\n" + "17,6\n" + "18,6\n" + "19,6\n" + "20,6\n" + "21,6\n";
- compareResultAsText(results, expected);
- }
-
- @Test
- public void testRegisterWithFields() throws Exception {
- final String tableName = "MyTable";
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = getJavaTableEnvironment();
- TranslationContext.reset();
-
- DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
- tableEnv.registerDataSet(tableName, ds, "a, b, c");
- Table t = tableEnv.scan(tableName);
-
- Table result = t.select("a, b, c");
-
- DataSet<Row> resultSet = tableEnv.toDataSet(result, Row.class);
- List<Row> results = resultSet.collect();
- String expected = "1,1,Hi\n" + "2,2,Hello\n" + "3,2,Hello world\n" +
- "4,3,Hello world, how are you?\n" + "5,3,I am fine.\n" + "6,3,Luke Skywalker\n" +
- "7,4,Comment#1\n" + "8,4,Comment#2\n" + "9,4,Comment#3\n" + "10,4,Comment#4\n" +
- "11,5,Comment#5\n" + "12,5,Comment#6\n" + "13,5,Comment#7\n" +
- "14,5,Comment#8\n" + "15,5,Comment#9\n" + "16,6,Comment#10\n" +
- "17,6,Comment#11\n" + "18,6,Comment#12\n" + "19,6,Comment#13\n" +
- "20,6,Comment#14\n" + "21,6,Comment#15\n";
- compareResultAsText(results, expected);
- }
-
- @Test(expected = TableException.class)
- public void testRegisterExistingDatasetTable() throws Exception {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = getJavaTableEnvironment();
- TranslationContext.reset();
-
- DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
- tableEnv.registerDataSet("MyTable", ds);
- DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 =
- CollectionDataSets.getSmall5TupleDataSet(env);
- tableEnv.registerDataSet("MyTable", ds2);
- }
-
- @Test(expected = TableException.class)
- public void testScanUnregisteredTable() throws Exception {
- TableEnvironment tableEnv = getJavaTableEnvironment();
- TranslationContext.reset();
-
- tableEnv.scan("nonRegisteredTable");
- }
-
- @Test
- public void testTableRegister() throws Exception {
- final String tableName = "MyTable";
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = getJavaTableEnvironment();
- TranslationContext.reset();
-
- DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
- Table t = tableEnv.fromDataSet(ds);
- tableEnv.registerTable(tableName, t);
- Table result = tableEnv.scan(tableName).select("f0, f1").filter("f0 > 7");
-
- DataSet<Row> resultSet = tableEnv.toDataSet(result, Row.class);
- List<Row> results = resultSet.collect();
- String expected = "8,4\n" + "9,4\n" + "10,4\n" + "11,5\n" + "12,5\n" +
- "13,5\n" + "14,5\n" + "15,5\n" +
- "16,6\n" + "17,6\n" + "18,6\n" + "19,6\n" + "20,6\n" + "21,6\n";
- compareResultAsText(results, expected);
- }
-
- @Test(expected = TableException.class)
- public void testIllegalName() throws Exception {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = getJavaTableEnvironment();
- TranslationContext.reset();
-
- DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
- Table t = tableEnv.fromDataSet(ds);
- tableEnv.registerTable("_DataSetTable_42", t);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/SelectITCase.java
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/SelectITCase.java b/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/SelectITCase.java
index 24cae60..5029808 100644
--- a/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/SelectITCase.java
+++ b/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/SelectITCase.java
@@ -22,8 +22,9 @@ import org.apache.flink.api.table.Table;
import org.apache.flink.api.table.Row;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.table.TableEnvironment;
+import org.apache.flink.api.java.table.BatchTableEnvironment;
import org.apache.flink.api.java.tuple.Tuple3;
+import org.apache.flink.api.table.TableEnvironment;
import org.apache.flink.api.table.test.utils.TableProgramsTestBase;
import org.apache.flink.test.javaApiOperators.util.CollectionDataSets;
import org.junit.Test;
@@ -42,7 +43,7 @@ public class SelectITCase extends TableProgramsTestBase {
@Test
public void testSimpleSelectAllWithAs() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = getJavaTableEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env, config());
DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
Table in = tableEnv.fromDataSet(ds, "a,b,c");
@@ -65,7 +66,7 @@ public class SelectITCase extends TableProgramsTestBase {
@Test
public void testSimpleSelectWithNaming() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = getJavaTableEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env, config());
DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
Table in = tableEnv.fromDataSet(ds);
@@ -85,7 +86,7 @@ public class SelectITCase extends TableProgramsTestBase {
@Test
public void testSimpleSelectRenameAll() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = getJavaTableEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env, config());
DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
Table in = tableEnv.fromDataSet(ds);
@@ -105,7 +106,7 @@ public class SelectITCase extends TableProgramsTestBase {
@Test(expected = IllegalArgumentException.class)
public void testSelectInvalidField() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = getJavaTableEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env, config());
DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
@@ -117,7 +118,7 @@ public class SelectITCase extends TableProgramsTestBase {
@Test(expected = IllegalArgumentException.class)
public void testSelectAmbiguousFieldNames() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = getJavaTableEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env, config());
DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/SqlExplainITCase.java
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/SqlExplainITCase.java b/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/SqlExplainITCase.java
deleted file mode 100644
index c985a37..0000000
--- a/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/SqlExplainITCase.java
+++ /dev/null
@@ -1,162 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.api.java.table.test;
-
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.table.TableEnvironment;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.api.table.Table;
-import org.apache.flink.api.table.plan.TranslationContext;
-import org.apache.flink.test.util.MultipleProgramsTestBase;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.File;
-import java.util.Scanner;
-
-import static org.junit.Assert.assertEquals;
-
-public class SqlExplainITCase extends MultipleProgramsTestBase {
-
- public SqlExplainITCase() {
- super(TestExecutionMode.CLUSTER);
- }
-
- private static String testFilePath = SqlExplainITCase.class.getResource("/").getFile();
-
- @Before
- public void resetContext() {
- TranslationContext.reset();
- }
-
- @Test
- public void testFilterWithoutExtended() throws Exception {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = new TableEnvironment();
-
- DataSet<Tuple2<Integer, String>> input = env.fromElements(new Tuple2<>(1,"d"));
- Table table = tableEnv.fromDataSet(input, "a, b");
-
- String result = table
- .filter("a % 2 = 0")
- .explain();
- String source = new Scanner(new File(testFilePath +
- "../../src/test/scala/resources/testFilter0.out"))
- .useDelimiter("\\A").next();
- assertEquals(source, result);
- }
-
- @Test
- public void testFilterWithExtended() throws Exception {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = new TableEnvironment();
-
- DataSet<Tuple2<Integer, String>> input = env.fromElements(new Tuple2<>(1,"d"));
- Table table = tableEnv.fromDataSet(input, "a, b");
-
- String result = table
- .filter("a % 2 = 0")
- .explain(true);
- String source = new Scanner(new File(testFilePath +
- "../../src/test/scala/resources/testFilter1.out"))
- .useDelimiter("\\A").next();
- assertEquals(source, result);
- }
-
- @Test
- public void testJoinWithoutExtended() throws Exception {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = new TableEnvironment();
-
- DataSet<Tuple2<Integer, String>> input1 = env.fromElements(new Tuple2<>(1,"d"));
- DataSet<Tuple2<Integer, String>> input2 = env.fromElements(new Tuple2<>(1,"d"));
- Table table1 = tableEnv.fromDataSet(input1, "a, b");
- Table table2 = tableEnv.fromDataSet(input2, "c, d");
-
- String result = table1
- .join(table2)
- .where("b = d")
- .select("a, c")
- .explain();
- String source = new Scanner(new File(testFilePath +
- "../../src/test/scala/resources/testJoin0.out"))
- .useDelimiter("\\A").next();
- assertEquals(source, result);
- }
-
- @Test
- public void testJoinWithExtended() throws Exception {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = new TableEnvironment();
-
- DataSet<Tuple2<Integer, String>> input1 = env.fromElements(new Tuple2<>(1,"d"));
- DataSet<Tuple2<Integer, String>> input2 = env.fromElements(new Tuple2<>(1,"d"));
- Table table1 = tableEnv.fromDataSet(input1, "a, b");
- Table table2 = tableEnv.fromDataSet(input2, "c, d");
-
- String result = table1
- .join(table2)
- .where("b = d")
- .select("a, c")
- .explain(true);
- String source = new Scanner(new File(testFilePath +
- "../../src/test/scala/resources/testJoin1.out"))
- .useDelimiter("\\A").next();
- assertEquals(source, result);
- }
-
- @Test
- public void testUnionWithoutExtended() throws Exception {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = new TableEnvironment();
-
- DataSet<Tuple2<Integer, String>> input1 = env.fromElements(new Tuple2<>(1,"d"));
- DataSet<Tuple2<Integer, String>> input2 = env.fromElements(new Tuple2<>(1,"d"));
- Table table1 = tableEnv.fromDataSet(input1, "count, word");
- Table table2 = tableEnv.fromDataSet(input2, "count, word");
-
- String result = table1
- .unionAll(table2)
- .explain();
- String source = new Scanner(new File(testFilePath +
- "../../src/test/scala/resources/testUnion0.out"))
- .useDelimiter("\\A").next();
- assertEquals(source, result);
- }
-
- @Test
- public void testUnionWithExtended() throws Exception {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = new TableEnvironment();
-
- DataSet<Tuple2<Integer, String>> input1 = env.fromElements(new Tuple2<>(1,"d"));
- DataSet<Tuple2<Integer, String>> input2 = env.fromElements(new Tuple2<>(1,"d"));
- Table table1 = tableEnv.fromDataSet(input1, "count, word");
- Table table2 = tableEnv.fromDataSet(input2, "count, word");
-
- String result = table1
- .unionAll(table2)
- .explain(true);
- String source = new Scanner(new File(testFilePath +
- "../../src/test/scala/resources/testUnion1.out"))
- .useDelimiter("\\A").next();
- assertEquals(source, result);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/SqlExplainTest.java
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/SqlExplainTest.java b/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/SqlExplainTest.java
new file mode 100644
index 0000000..74cd802
--- /dev/null
+++ b/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/SqlExplainTest.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.java.table.test;
+
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.table.BatchTableEnvironment;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.table.Table;
+import org.apache.flink.api.table.TableEnvironment;
+import org.apache.flink.test.util.MultipleProgramsTestBase;
+import org.junit.Test;
+
+import java.io.File;
+import java.util.Scanner;
+
+import static org.junit.Assert.assertEquals;
+
+public class SqlExplainTest extends MultipleProgramsTestBase {
+
+ public SqlExplainTest() {
+ super(TestExecutionMode.CLUSTER);
+ }
+
+ private static String testFilePath = SqlExplainTest.class.getResource("/").getFile();
+
+ @Test
+ public void testFilterWithoutExtended() throws Exception {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
+
+ DataSet<Tuple2<Integer, String>> input = env.fromElements(new Tuple2<>(1,"d"));
+ Table table = tableEnv
+ .fromDataSet(input, "a, b")
+ .filter("a % 2 = 0");
+
+ String result = tableEnv.explain(table);
+ String source = new Scanner(new File(testFilePath +
+ "../../src/test/scala/resources/testFilter0.out"))
+ .useDelimiter("\\A").next();
+ assertEquals(source, result);
+ }
+
+ @Test
+ public void testFilterWithExtended() throws Exception {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
+
+ DataSet<Tuple2<Integer, String>> input = env.fromElements(new Tuple2<>(1,"d"));
+ Table table = tableEnv
+ .fromDataSet(input, "a, b")
+ .filter("a % 2 = 0");
+
+ String result = tableEnv.explain(table, true);
+ String source = new Scanner(new File(testFilePath +
+ "../../src/test/scala/resources/testFilter1.out"))
+ .useDelimiter("\\A").next();
+ assertEquals(source, result);
+ }
+
+ @Test
+ public void testJoinWithoutExtended() throws Exception {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
+
+ DataSet<Tuple2<Integer, String>> input1 = env.fromElements(new Tuple2<>(1,"d"));
+ DataSet<Tuple2<Integer, String>> input2 = env.fromElements(new Tuple2<>(1,"d"));
+ Table table1 = tableEnv.fromDataSet(input1, "a, b");
+ Table table2 = tableEnv.fromDataSet(input2, "c, d");
+ Table table = table1
+ .join(table2)
+ .where("b = d")
+ .select("a, c");
+
+ String result = tableEnv.explain(table);
+ String source = new Scanner(new File(testFilePath +
+ "../../src/test/scala/resources/testJoin0.out"))
+ .useDelimiter("\\A").next();
+ assertEquals(source, result);
+ }
+
+ @Test
+ public void testJoinWithExtended() throws Exception {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
+
+ DataSet<Tuple2<Integer, String>> input1 = env.fromElements(new Tuple2<>(1,"d"));
+ DataSet<Tuple2<Integer, String>> input2 = env.fromElements(new Tuple2<>(1,"d"));
+ Table table1 = tableEnv.fromDataSet(input1, "a, b");
+ Table table2 = tableEnv.fromDataSet(input2, "c, d");
+ Table table = table1
+ .join(table2)
+ .where("b = d")
+ .select("a, c");
+
+ String result = tableEnv.explain(table, true);
+ String source = new Scanner(new File(testFilePath +
+ "../../src/test/scala/resources/testJoin1.out"))
+ .useDelimiter("\\A").next();
+ assertEquals(source, result);
+ }
+
+ @Test
+ public void testUnionWithoutExtended() throws Exception {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
+
+ DataSet<Tuple2<Integer, String>> input1 = env.fromElements(new Tuple2<>(1,"d"));
+ DataSet<Tuple2<Integer, String>> input2 = env.fromElements(new Tuple2<>(1,"d"));
+ Table table1 = tableEnv.fromDataSet(input1, "count, word");
+ Table table2 = tableEnv.fromDataSet(input2, "count, word");
+ Table table = table1.unionAll(table2);
+
+ String result = tableEnv.explain(table);
+ String source = new Scanner(new File(testFilePath +
+ "../../src/test/scala/resources/testUnion0.out"))
+ .useDelimiter("\\A").next();
+ assertEquals(source, result);
+ }
+
+ @Test
+ public void testUnionWithExtended() throws Exception {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
+
+ DataSet<Tuple2<Integer, String>> input1 = env.fromElements(new Tuple2<>(1,"d"));
+ DataSet<Tuple2<Integer, String>> input2 = env.fromElements(new Tuple2<>(1,"d"));
+ Table table1 = tableEnv.fromDataSet(input1, "count, word");
+ Table table2 = tableEnv.fromDataSet(input2, "count, word");
+ Table table = table1.unionAll(table2);
+
+ String result = tableEnv.explain(table, true);
+ String source = new Scanner(new File(testFilePath +
+ "../../src/test/scala/resources/testUnion1.out"))
+ .useDelimiter("\\A").next();
+ assertEquals(source, result);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/1acd8445/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/StringExpressionsITCase.java
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/StringExpressionsITCase.java b/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/StringExpressionsITCase.java
index 86a3bfd..32afc5d 100644
--- a/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/StringExpressionsITCase.java
+++ b/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/table/test/StringExpressionsITCase.java
@@ -20,13 +20,14 @@ package org.apache.flink.api.java.table.test;
import org.apache.flink.api.java.tuple.Tuple3;
+import org.apache.flink.api.table.TableEnvironment;
import org.apache.flink.test.javaApiOperators.util.CollectionDataSets;
import org.apache.flink.api.table.Row;
import org.apache.flink.api.table.Table;
import org.apache.flink.api.table.codegen.CodeGenException;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.table.TableEnvironment;
+import org.apache.flink.api.java.table.BatchTableEnvironment;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.test.util.MultipleProgramsTestBase;
import org.junit.Test;
@@ -45,7 +46,7 @@ public class StringExpressionsITCase extends MultipleProgramsTestBase {
@Test
public void testSubstring() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = new TableEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
DataSet<Tuple2<String, Integer>> ds = env.fromElements(
new Tuple2<>("AAAA", 2),
@@ -65,7 +66,7 @@ public class StringExpressionsITCase extends MultipleProgramsTestBase {
@Test
public void testSubstringWithMaxEnd() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = new TableEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
DataSet<Tuple2<String, Integer>> ds = env.fromElements(
new Tuple2<>("ABCD", 3),
@@ -85,7 +86,7 @@ public class StringExpressionsITCase extends MultipleProgramsTestBase {
@Test(expected = CodeGenException.class)
public void testNonWorkingSubstring1() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = new TableEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
DataSet<Tuple2<String, Float>> ds = env.fromElements(
new Tuple2<>("ABCD", 2.0f),
@@ -104,7 +105,7 @@ public class StringExpressionsITCase extends MultipleProgramsTestBase {
@Test(expected = CodeGenException.class)
public void testNonWorkingSubstring2() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = new TableEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
DataSet<Tuple2<String, String>> ds = env.fromElements(
new Tuple2<>("ABCD", "a"),
@@ -123,7 +124,7 @@ public class StringExpressionsITCase extends MultipleProgramsTestBase {
@Test(expected = CodeGenException.class)
public void testGeneratedCodeForStringComparison() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = new TableEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
DataSet<Tuple3<Integer, Long, String>> tupleDataSet = CollectionDataSets.get3TupleDataSet(env);
Table in = tableEnv.fromDataSet(tupleDataSet, "a, b, c");
// Must fail because the comparison here is between Integer(column 'a') and (String 'Fred')
@@ -134,7 +135,7 @@ public class StringExpressionsITCase extends MultipleProgramsTestBase {
@Test(expected = CodeGenException.class)
public void testGeneratedCodeForIntegerEqualsComparison() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = new TableEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
DataSet<Tuple3<Integer, Long, String>> tupleDataSet = CollectionDataSets.get3TupleDataSet(env);
Table in = tableEnv.fromDataSet(tupleDataSet, "a, b, c");
// Must fail because the comparison here is between String(column 'c') and (Integer 10)
@@ -145,7 +146,7 @@ public class StringExpressionsITCase extends MultipleProgramsTestBase {
@Test(expected = CodeGenException.class)
public void testGeneratedCodeForIntegerGreaterComparison() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = new TableEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
DataSet<Tuple3<Integer, Long, String>> tupleDataSet = CollectionDataSets.get3TupleDataSet(env);
Table in = tableEnv.fromDataSet(tupleDataSet, "a, b, c");
// Must fail because the comparison here is between String(column 'c') and (Integer 10)
@@ -156,7 +157,7 @@ public class StringExpressionsITCase extends MultipleProgramsTestBase {
@Test
public void testStringConcat() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = new TableEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
DataSet<Tuple2<String, Integer>> ds = env.fromElements(
new Tuple2<>("ABCD", 3),
@@ -176,7 +177,7 @@ public class StringExpressionsITCase extends MultipleProgramsTestBase {
@Test
public void testStringConcat1() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- TableEnvironment tableEnv = new TableEnvironment();
+ BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
DataSet<Tuple2<String, Integer>> ds = env.fromElements(
new Tuple2<>("ABCD", 3),