You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by fh...@apache.org on 2017/10/24 18:29:06 UTC
[3/3] flink git commit: [FLINK-7821] [table] Deprecate Table.limit()
and replace it by Table.offset() and Table.fetch().
[FLINK-7821] [table] Deprecate Table.limit() and replace it by Table.offset() and Table.fetch().
This closes #4813.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/e1c7e490
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/e1c7e490
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/e1c7e490
Branch: refs/heads/master
Commit: e1c7e490395c44913e816966b2fa3b8bfeb18160
Parents: 7da886a
Author: Fabian Hueske <fh...@apache.org>
Authored: Thu Oct 12 10:56:19 2017 +0200
Committer: Fabian Hueske <fh...@apache.org>
Committed: Tue Oct 24 20:28:46 2017 +0200
----------------------------------------------------------------------
docs/dev/table/tableApi.md | 42 +++++++------
.../org/apache/flink/table/api/table.scala | 66 +++++++++++++++++++-
.../table/validation/SortValidationTest.scala | 34 +++++++++-
.../validation/CorrelateValidationTest.scala | 4 +-
.../UnsupportedOpsValidationTest.scala | 12 +++-
.../table/runtime/batch/table/SortITCase.scala | 6 +-
6 files changed, 135 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/e1c7e490/docs/dev/table/tableApi.md
----------------------------------------------------------------------
diff --git a/docs/dev/table/tableApi.md b/docs/dev/table/tableApi.md
index 07b301d..f0c4605 100644
--- a/docs/dev/table/tableApi.md
+++ b/docs/dev/table/tableApi.md
@@ -959,7 +959,7 @@ val result = left.select('a, 'b, 'c).where('a.in(right));
{% top %}
-### OrderBy & Limit
+### OrderBy, Offset & Fetch
<div class="codetabs" markdown="1">
<div data-lang="java" markdown="1">
@@ -987,19 +987,22 @@ Table result = in.orderBy("a.asc");
<tr>
<td>
- <strong>Limit</strong><br>
+ <strong>Offset & Fetch</strong><br>
<span class="label label-primary">Batch</span>
</td>
<td>
- <p>Similar to a SQL LIMIT clause. Limits a sorted result to a specified number of records from an offset position. Limit is technically part of the Order By operator and thus must be preceded by it.</p>
+ <p>Similar to the SQL OFFSET and FETCH clauses. Offset and Fetch limit the number of records returned from a sorted result. Offset and Fetch are technically part of the Order By operator and thus must be preceded by it.</p>
{% highlight java %}
Table in = tableEnv.fromDataSet(ds, "a, b, c");
-Table result = in.orderBy("a.asc").limit(3); // returns unlimited number of records beginning with the 4th record
-{% endhighlight %}
-or
-{% highlight java %}
-Table in = tableEnv.fromDataSet(ds, "a, b, c");
-Table result = in.orderBy("a.asc").limit(3, 5); // returns 5 records beginning with the 4th record
+
+// returns the first 5 records from the sorted result
+Table result1 = in.orderBy("a.asc").fetch(5);
+
+// skips the first 3 records and returns all following records from the sorted result
+Table result2 = in.orderBy("a.asc").offset(3);
+
+// skips the first 10 records and returns the next 5 records from the sorted result
+Table result3 = in.orderBy("a.asc").offset(10).fetch(5);
{% endhighlight %}
</td>
</tr>
@@ -1033,19 +1036,22 @@ val result = in.orderBy('a.asc);
<tr>
<td>
- <strong>Limit</strong><br>
+ <strong>Offset & Fetch</strong><br>
<span class="label label-primary">Batch</span>
</td>
<td>
- <p>Similar to a SQL LIMIT clause. Limits a sorted result to a specified number of records from an offset position. Limit is technically part of the Order By operator and thus must be preceded by it.</p>
+ <p>Similar to the SQL OFFSET and FETCH clauses. Offset and Fetch limit the number of records returned from a sorted result. Offset and Fetch are technically part of the Order By operator and thus must be preceded by it.</p>
{% highlight scala %}
-val in = ds.toTable(tableEnv, 'a, 'b, 'c);
-val result = in.orderBy('a.asc).limit(3); // returns unlimited number of records beginning with the 4th record
-{% endhighlight %}
-or
-{% highlight scala %}
-val in = ds.toTable(tableEnv, 'a, 'b, 'c);
-val result = in.orderBy('a.asc).limit(3, 5); // returns 5 records beginning with the 4th record
+val in = ds.toTable(tableEnv, 'a, 'b, 'c)
+
+// returns the first 5 records from the sorted result
+val result1: Table = in.orderBy('a.asc).fetch(5)
+
+// skips the first 3 records and returns all following records from the sorted result
+val result2: Table = in.orderBy('a.asc).offset(3)
+
+// skips the first 10 records and returns the next 5 records from the sorted result
+val result3: Table = in.orderBy('a.asc).offset(10).fetch(5)
{% endhighlight %}
</td>
</tr>
http://git-wip-us.apache.org/repos/asf/flink/blob/e1c7e490/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/table.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/table.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/table.scala
index 30ed98e..0430e49 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/table.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/table.scala
@@ -25,7 +25,6 @@ import org.apache.flink.table.expressions.{Alias, Asc, Expression, ExpressionPar
import org.apache.flink.table.functions.utils.UserDefinedFunctionUtils
import org.apache.flink.table.plan.ProjectionTranslator._
import org.apache.flink.table.plan.logical.{Minus, _}
-import org.apache.flink.table.plan.schema.TableSinkTable
import org.apache.flink.table.sinks.TableSink
import _root_.scala.annotation.varargs
@@ -721,12 +720,16 @@ class Table(
* Example:
*
* {{{
- * // returns unlimited number of records beginning with the 4th record
+ * // skips the first 3 rows and returns all following rows.
* tab.orderBy('name.desc).limit(3)
* }}}
*
* @param offset number of records to skip
+ *
+ * @deprecated Please use [[Table.offset()]] and [[Table.fetch()]] instead.
*/
+ @Deprecated
+ @deprecated(message = "Deprecated in favor of Table.offset() and Table.fetch()", since = "1.4.0")
def limit(offset: Int): Table = {
new Table(tableEnv, Limit(offset = offset, child = logicalPlan).validate(tableEnv))
}
@@ -739,18 +742,75 @@ class Table(
* Example:
*
* {{{
- * // returns 5 records beginning with the 4th record
+ * // skips the first 3 rows and returns the next 5 rows.
* tab.orderBy('name.desc).limit(3, 5)
* }}}
*
* @param offset number of records to skip
* @param fetch number of records to be returned
+ *
+ * @deprecated Please use [[Table.offset()]] and [[Table.fetch()]] instead.
*/
+ @Deprecated
+ @deprecated(message = "deprecated in favor of Table.offset() and Table.fetch()", since = "1.4.0")
def limit(offset: Int, fetch: Int): Table = {
new Table(tableEnv, Limit(offset, fetch, logicalPlan).validate(tableEnv))
}
/**
+ * Limits a sorted result from an offset position.
+ * Similar to a SQL OFFSET clause. Offset is technically part of the Order By operator and
+ * thus must be preceded by it.
+ *
+ * [[Table.offset(o)]] can be combined with a subsequent [[Table.fetch(n)]] call to return n rows
+ * after skipping the first o rows.
+ *
+ * {{{
+ * // skips the first 3 rows and returns all following rows.
+ * tab.orderBy('name.desc).offset(3)
+ * // skips the first 10 rows and returns the next 5 rows.
+ * tab.orderBy('name.desc).offset(10).fetch(5)
+ * }}}
+ *
+ * @param offset number of records to skip
+ */
+ def offset(offset: Int): Table = {
+ new Table(tableEnv, Limit(offset, -1, logicalPlan).validate(tableEnv))
+ }
+
+ /**
+ * Limits a sorted result to the first n rows.
+ * Similar to a SQL FETCH clause. Fetch is technically part of the Order By operator and
+ * thus must be preceded by it.
+ *
+ * [[Table.fetch(n)]] can be combined with a preceding [[Table.offset(o)]] call to return n rows
+ * after skipping the first o rows.
+ *
+ * {{{
+ * // returns the first 3 records.
+ * tab.orderBy('name.desc).fetch(3)
+ * // skips the first 10 rows and returns the next 5 rows.
+ * tab.orderBy('name.desc).offset(10).fetch(5)
+ * }}}
+ *
+ * @param fetch the number of records to return. Fetch must be >= 0.
+ */
+ def fetch(fetch: Int): Table = {
+ if (fetch < 0) {
+ throw ValidationException("FETCH count must be equal or larger than 0.")
+ }
+ this.logicalPlan match {
+ case Limit(o, -1, c) =>
+ // replace LIMIT without FETCH by LIMIT with FETCH
+ new Table(tableEnv, Limit(o, fetch, c).validate(tableEnv))
+ case Limit(_, _, _) =>
+ throw ValidationException("FETCH is already defined.")
+ case _ =>
+ new Table(tableEnv, Limit(0, fetch, logicalPlan).validate(tableEnv))
+ }
+ }
+
+ /**
* Writes the [[Table]] to a [[TableSink]]. A [[TableSink]] defines an external storage location.
*
* A batch [[Table]] can only be written to a
http://git-wip-us.apache.org/repos/asf/flink/blob/e1c7e490/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/batch/table/validation/SortValidationTest.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/batch/table/validation/SortValidationTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/batch/table/validation/SortValidationTest.scala
index 5064687..24156e4 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/batch/table/validation/SortValidationTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/batch/table/validation/SortValidationTest.scala
@@ -27,10 +27,42 @@ import org.junit._
class SortValidationTest extends TableTestBase {
@Test(expected = classOf[ValidationException])
+ def testOffsetWithoutOrder(): Unit = {
+ val util = batchTestUtil()
+ val ds = util.addTable[(Int, Long, String)]("Table3", 'a, 'b, 'c)
+
+ ds.offset(5)
+ }
+
+ @Test(expected = classOf[ValidationException])
def testFetchWithoutOrder(): Unit = {
val util = batchTestUtil()
val ds = util.addTable[(Int, Long, String)]("Table3", 'a, 'b, 'c)
- ds.limit(0, 5)
+ ds.fetch(5)
+ }
+
+ @Test(expected = classOf[ValidationException])
+ def testFetchBeforeOffset(): Unit = {
+ val util = batchTestUtil()
+ val ds = util.addTable[(Int, Long, String)]("Table3", 'a, 'b, 'c)
+
+ ds.orderBy('a.asc).fetch(5).offset(10)
+ }
+
+ @Test(expected = classOf[ValidationException])
+ def testOffsetBeforeOffset(): Unit = {
+ val util = batchTestUtil()
+ val ds = util.addTable[(Int, Long, String)]("Table3", 'a, 'b, 'c)
+
+ ds.orderBy('a.asc).offset(10).offset(5)
+ }
+
+ @Test(expected = classOf[ValidationException])
+ def testNegativeFetch(): Unit = {
+ val util = batchTestUtil()
+ val ds = util.addTable[(Int, Long, String)]("Table3", 'a, 'b, 'c)
+
+ ds.orderBy('a.asc).offset(-1)
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/e1c7e490/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/stream/table/validation/CorrelateValidationTest.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/stream/table/validation/CorrelateValidationTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/stream/table/validation/CorrelateValidationTest.scala
index c754afd..8f429e1 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/stream/table/validation/CorrelateValidationTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/stream/table/validation/CorrelateValidationTest.scala
@@ -86,13 +86,13 @@ class CorrelateValidationTest extends TableTestBase {
// table function call limit
expectExceptionThrown(
- func1('c).orderBy('f0).limit(3),
+ func1('c).orderBy('f0).offset(3),
"TableFunction can only be used in join and leftOuterJoin."
)
// table function call limit
expectExceptionThrown(
- func1('c).orderBy('f0).limit(0, 3),
+ func1('c).orderBy('f0).fetch(3),
"TableFunction can only be used in join and leftOuterJoin."
)
http://git-wip-us.apache.org/repos/asf/flink/blob/e1c7e490/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/stream/table/validation/UnsupportedOpsValidationTest.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/stream/table/validation/UnsupportedOpsValidationTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/stream/table/validation/UnsupportedOpsValidationTest.scala
index de7f70a..c1ad08c 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/stream/table/validation/UnsupportedOpsValidationTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/stream/table/validation/UnsupportedOpsValidationTest.scala
@@ -89,11 +89,19 @@ class UnsupportedOpsValidationTest extends StreamingMultipleProgramsTestBase {
}
@Test(expected = classOf[ValidationException])
- def testLimit(): Unit = {
+ def testOffset(): Unit = {
val env = StreamExecutionEnvironment.getExecutionEnvironment
val tEnv = TableEnvironment.getTableEnvironment(env)
val t1 = StreamTestData.getSmall3TupleDataStream(env).toTable(tEnv)
- t1.limit(0,5)
+ t1.offset(5)
+ }
+
+ @Test(expected = classOf[ValidationException])
+ def testFetch(): Unit = {
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+ val t1 = StreamTestData.getSmall3TupleDataStream(env).toTable(tEnv)
+ t1.fetch(5)
}
@Test(expected = classOf[ValidationException])
http://git-wip-us.apache.org/repos/asf/flink/blob/e1c7e490/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/batch/table/SortITCase.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/batch/table/SortITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/batch/table/SortITCase.scala
index 568fad0..79bb711 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/batch/table/SortITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/batch/table/SortITCase.scala
@@ -143,7 +143,7 @@ class SortITCase(mode: TestExecutionMode, configMode: TableConfigMode)
val tEnv = TableEnvironment.getTableEnvironment(env, config)
val ds = CollectionDataSets.get3TupleDataSet(env)
- val t = ds.toTable(tEnv).orderBy('_1.asc).limit(3)
+ val t = ds.toTable(tEnv).orderBy('_1.asc).offset(3)
implicit def tupleOrdering[T <: Product] = Ordering.by((x : T) =>
x.productElement(0).asInstanceOf[Int] )
@@ -171,7 +171,7 @@ class SortITCase(mode: TestExecutionMode, configMode: TableConfigMode)
val tEnv = TableEnvironment.getTableEnvironment(env, config)
val ds = CollectionDataSets.get3TupleDataSet(env)
- val t = ds.toTable(tEnv).orderBy('_1.desc).limit(3, 5)
+ val t = ds.toTable(tEnv).orderBy('_1.desc).offset(3).fetch(5)
implicit def tupleOrdering[T <: Product] = Ordering.by((x : T) =>
- x.productElement(0).asInstanceOf[Int] )
@@ -199,7 +199,7 @@ class SortITCase(mode: TestExecutionMode, configMode: TableConfigMode)
val tEnv = TableEnvironment.getTableEnvironment(env, config)
val ds = CollectionDataSets.get3TupleDataSet(env)
- val t = ds.toTable(tEnv).orderBy('_1.asc).limit(0, 5)
+ val t = ds.toTable(tEnv).orderBy('_1.asc).fetch(5)
implicit def tupleOrdering[T <: Product] = Ordering.by((x : T) =>
x.productElement(0).asInstanceOf[Int] )